github/kubernetes
1
0
Fork 0
mirror of https://github.com/k3s-io/kubernetes.git synced 2026-07-18 13:57:38 +00:00
Code Issues Projects Releases Wiki Activity

Account for node-allocatable claims in NodeInfo cache

Browse Source
This commit is contained in:
Praveen Krishna
2026-02-02 20:44:11 +00:00
parent 0c25bdf74d
commit 7c4f9ace63
3 changed files with 637 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
pkg/scheduler/framework/types.go
View File

@@ -26,6 +26,7 @@ import (
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
@@ -205,6 +206,11 @@ type NodeInfo struct {
// DeclaredFeatures is a set of features published by the node
DeclaredFeatures ndf.FeatureSet
// NodeAllocatableDRAClaimStates tracks the state of claims requesting node-allocatable resources
// (resources published in Node.Status.Allocatable like cpu, memory. etc.).
// This is used to enforce sharing policies for these claims on the node.
NodeAllocatableDRAClaimStates map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState
}
func (n *NodeInfo) GetPods() []fwk.PodInfo {
@@ -252,6 +258,10 @@ func (n *NodeInfo) GetNodeDeclaredFeatures() ndf.FeatureSet {
return n.DeclaredFeatures
}
func (n *NodeInfo) GetNodeAllocatableDRAClaimState() map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState {
return n.NodeAllocatableDRAClaimStates
}
// NodeInfo implements KMetadata, so for example klog.KObjSlice(nodes) works
// when nodes is a []*NodeInfo.
var _ klog.KMetadata = &NodeInfo{}
@@ -290,15 +300,16 @@ func (n *NodeInfo) Snapshot() fwk.NodeInfo {
// SnapshotConcrete returns a copy of this node, Except that ImageStates is copied without the Nodes field.
func (n *NodeInfo) SnapshotConcrete() *NodeInfo {
clone := &NodeInfo{
node: n.node,
Requested: n.Requested.Clone(),
NonZeroRequested: n.NonZeroRequested.Clone(),
Allocatable: n.Allocatable.Clone(),
UsedPorts: make(fwk.HostPortInfo),
ImageStates: make(map[string]*fwk.ImageStateSummary),
PVCRefCounts: make(map[string]int),
Generation: n.Generation,
DeclaredFeatures: n.DeclaredFeatures.Clone(),
node: n.node,
Requested: n.Requested.Clone(),
NonZeroRequested: n.NonZeroRequested.Clone(),
Allocatable: n.Allocatable.Clone(),
UsedPorts: make(fwk.HostPortInfo),
ImageStates: make(map[string]*fwk.ImageStateSummary),
PVCRefCounts: make(map[string]int),
Generation: n.Generation,
DeclaredFeatures: n.DeclaredFeatures.Clone(),
NodeAllocatableDRAClaimStates: make(map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState),
}
if len(n.Pods) > 0 {
clone.Pods = append([]fwk.PodInfo(nil), n.Pods...)
@@ -329,6 +340,9 @@ func (n *NodeInfo) SnapshotConcrete() *NodeInfo {
for key, value := range n.PVCRefCounts {
clone.PVCRefCounts[key] = value
}
for key, value := range n.NodeAllocatableDRAClaimStates {
clone.NodeAllocatableDRAClaimStates[key] = value.Snapshot()
}
return clone
}
@@ -441,6 +455,42 @@ func (n *NodeInfo) update(podInfo fwk.PodInfo, sign int64) {
n.updatePVCRefCounts(podInfo.GetPod(), sign > 0)
n.Generation = nextGeneration()
if utilfeature.DefaultFeatureGate.Enabled(features.DRANodeAllocatableResources) {
n.updateNodeAllocatableDRAClaimState(podInfo, sign)
}
}
// updateNodeAllocatableDRAClaimState updates the NodeInfo based on DRA node allocatable resource claims in the pod.
func (n *NodeInfo) updateNodeAllocatableDRAClaimState(podInfo fwk.PodInfo, sign int64) {
pod := podInfo.GetPod()
if n.NodeAllocatableDRAClaimStates == nil && len(pod.Status.NodeAllocatableResourceClaimStatuses) > 0 {
n.NodeAllocatableDRAClaimStates = make(map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState, len(pod.Status.NodeAllocatableResourceClaimStatuses))
}
for _, claimStatus := range pod.Status.NodeAllocatableResourceClaimStatuses {
resourceClaimNamespacedName := types.NamespacedName{
Namespace: pod.Namespace,
Name: claimStatus.ResourceClaimName,
}
if _, exists := n.NodeAllocatableDRAClaimStates[resourceClaimNamespacedName]; !exists {
n.NodeAllocatableDRAClaimStates[resourceClaimNamespacedName] = &fwk.NodeAllocatableDRAClaimState{
ConsumerPods: sets.New[types.UID](),
}
}
state := n.NodeAllocatableDRAClaimStates[resourceClaimNamespacedName]
if sign > 0 {
state.ConsumerPods.Insert(pod.UID)
} else {
state.ConsumerPods.Delete(pod.UID)
if state.ConsumerPods.Len() == 0 {
delete(n.NodeAllocatableDRAClaimStates, resourceClaimNamespacedName)
}
}
}
}
// updateUsedPorts updates the UsedPorts of NodeInfo.
@@ -786,11 +836,14 @@ func (pi *PodInfo) CalculateResource() fwk.PodResource {
inPlacePodVerticalScalingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling)
podLevelResourcesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources)
inPlacePodLevelResourcesVerticalScalingEnabled := utilfeature.DefaultMutableFeatureGate.Enabled(features.InPlacePodLevelResourcesVerticalScaling)
nodeAllocatableResourcesDRAEnabled := utilfeature.DefaultFeatureGate.Enabled(features.DRANodeAllocatableResources)
requests := resourcehelper.PodRequests(pi.Pod, resourcehelper.PodResourcesOptions{
UseStatusResources: inPlacePodVerticalScalingEnabled,
InPlacePodLevelResourcesVerticalScalingEnabled: inPlacePodLevelResourcesVerticalScalingEnabled,
// SkipPodLevelResources is set to false when PodLevelResources feature is enabled.
SkipPodLevelResources: !podLevelResourcesEnabled,
SkipPodLevelResources: !podLevelResourcesEnabled,
UseDRANodeAllocatableResourceClaimStatus: nodeAllocatableResourcesDRAEnabled,
})
isPodLevelResourcesSet := podLevelResourcesEnabled && resourcehelper.IsPodLevelRequestsSet(pi.Pod)
nonMissingContainerRequests := getNonMissingContainerRequests(requests, isPodLevelResourcesSet)
@@ -800,8 +853,9 @@ func (pi *PodInfo) CalculateResource() fwk.PodResource {
UseStatusResources: inPlacePodVerticalScalingEnabled,
InPlacePodLevelResourcesVerticalScalingEnabled: inPlacePodLevelResourcesVerticalScalingEnabled,
// SkipPodLevelResources is set to false when PodLevelResources feature is enabled.
SkipPodLevelResources: !podLevelResourcesEnabled,
NonMissingContainerRequests: nonMissingContainerRequests,
SkipPodLevelResources: !podLevelResourcesEnabled,
NonMissingContainerRequests: nonMissingContainerRequests,
UseDRANodeAllocatableResourceClaimStatus: nodeAllocatableResourcesDRAEnabled,
})
}
non0CPU := non0Requests[v1.ResourceCPU]
@@ -1052,13 +1106,14 @@ func (r *Resource) SetMaxResource(rl v1.ResourceList) {
// the returned object.
func NewNodeInfo(pods ...*v1.Pod) *NodeInfo {
ni := &NodeInfo{
Requested: &Resource{},
NonZeroRequested: &Resource{},
Allocatable: &Resource{},
Generation: nextGeneration(),
UsedPorts: make(fwk.HostPortInfo),
ImageStates: make(map[string]*fwk.ImageStateSummary),
PVCRefCounts: make(map[string]int),
Requested: &Resource{},
NonZeroRequested: &Resource{},
Allocatable: &Resource{},
Generation: nextGeneration(),
UsedPorts: make(fwk.HostPortInfo),
ImageStates: make(map[string]*fwk.ImageStateSummary),
PVCRefCounts: make(map[string]int),
NodeAllocatableDRAClaimStates: make(map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState),
}
for _, pod := range pods {
ni.AddPod(pod)

571
pkg/scheduler/framework/types_test.go
View File

@@ -22,6 +22,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"k8s.io/apimachinery/pkg/util/sets"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
@@ -270,8 +271,9 @@ func TestNewNodeInfo(t *testing.T) {
{Protocol: "TCP", Port: 8080}: {},
},
},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
Pods: []fwk.PodInfo{
&PodInfo{
Pod: &v1.Pod{
@@ -381,8 +383,9 @@ func TestNodeInfoClone(t *testing.T) {
{Protocol: "TCP", Port: 8080}: {},
},
},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
Pods: []fwk.PodInfo{
&PodInfo{
Pod: &v1.Pod{
@@ -471,8 +474,9 @@ func TestNodeInfoClone(t *testing.T) {
{Protocol: "TCP", Port: 8080}: {},
},
},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
Pods: []fwk.PodInfo{
&PodInfo{
Pod: &v1.Pod{
@@ -551,6 +555,30 @@ func TestNodeInfoClone(t *testing.T) {
},
},
},
{
nodeInfo: &NodeInfo{
Requested: &Resource{},
NonZeroRequested: &Resource{},
Allocatable: &Resource{},
Generation: 3,
UsedPorts: fwk.HostPortInfo{},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
DeclaredFeatures: declaredFeatureSet.Clone(),
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
},
expected: &NodeInfo{
Requested: &Resource{},
NonZeroRequested: &Resource{},
Allocatable: &Resource{},
Generation: 3,
UsedPorts: fwk.HostPortInfo{},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
DeclaredFeatures: declaredFeatureSet,
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
},
},
{
nodeInfo: &NodeInfo{
Requested: &Resource{},
@@ -561,6 +589,9 @@ func TestNodeInfoClone(t *testing.T) {
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
DeclaredFeatures: declaredFeatureSet.Clone(),
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
{Name: "claim1", Namespace: "default"}: {ConsumerPods: sets.New[types.UID]("pod1uid", "pod2uid")},
},
},
expected: &NodeInfo{
Requested: &Resource{},
@@ -571,6 +602,9 @@ func TestNodeInfoClone(t *testing.T) {
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
DeclaredFeatures: declaredFeatureSet,
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
{Name: "claim1", Namespace: "default"}: {ConsumerPods: sets.New[types.UID]("pod1uid", "pod2uid")},
},
},
},
}
@@ -747,9 +781,10 @@ func TestNodeInfoAddPod(t *testing.T) {
{Protocol: "TCP", Port: 8080}: {},
},
},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{"node_info_cache_test/pvc-1": 2, "node_info_cache_test/pvc-2": 1},
DeclaredFeatures: ndf.DefaultFramework.NewFeatureSet(), // Empty FeatureSet.
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{"node_info_cache_test/pvc-1": 2, "node_info_cache_test/pvc-2": 1},
DeclaredFeatures: ndf.DefaultFramework.NewFeatureSet(), // Empty FeatureSet.
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
Pods: []fwk.PodInfo{
&PodInfo{
Pod: &v1.Pod{
@@ -998,9 +1033,10 @@ func TestNodeInfoRemovePod(t *testing.T) {
{Protocol: "TCP", Port: 8080}: {},
},
},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{"node_info_cache_test/pvc-1": 1},
DeclaredFeatures: ndf.DefaultFramework.NewFeatureSet(), // Empty FeatureSet.
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{"node_info_cache_test/pvc-1": 1},
DeclaredFeatures: ndf.DefaultFramework.NewFeatureSet(), // Empty FeatureSet.
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
Pods: []fwk.PodInfo{
&PodInfo{
Pod: &v1.Pod{
@@ -1165,9 +1201,10 @@ func TestNodeInfoRemovePod(t *testing.T) {
{Protocol: "TCP", Port: 8080}: {},
},
},
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
DeclaredFeatures: ndf.DefaultFramework.NewFeatureSet(), // Empty FeatureSet.
ImageStates: map[string]*fwk.ImageStateSummary{},
PVCRefCounts: map[string]int{},
DeclaredFeatures: ndf.DefaultFramework.NewFeatureSet(), // Empty FeatureSet.
NodeAllocatableDRAClaimStates: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
Pods: []fwk.PodInfo{
&PodInfo{
Pod: &v1.Pod{
@@ -1470,10 +1507,13 @@ func TestFitError_Error(t *testing.T) {
}
var (
cpu100m = resource.MustParse("100m")
mem200M = resource.MustParse("200Mi")
cpu500m = resource.MustParse("500m")
mem500M = resource.MustParse("500Mi")
cpu700m = resource.MustParse("700m")
mem800M = resource.MustParse("800Mi")
cpu1000m = resource.MustParse("1000m")
cpu1200m = resource.MustParse("1200m")
mem1200M = resource.MustParse("1200Mi")
restartAlways = v1.ContainerRestartPolicyAlways
@@ -1481,12 +1521,15 @@ var (
func TestPodInfoCalculateResources(t *testing.T) {
testCases := []struct {
name string
containers []v1.Container
podResources *v1.ResourceRequirements
podLevelResourcesEnabled bool
expectedResource fwk.PodResource
initContainers []v1.Container
name string
containers []v1.Container
podResources *v1.ResourceRequirements
podLevelResourcesEnabled bool
nodeAllocatableResourcesDRAEnabled bool
nodeAllocatableResourceClaimStatuses []v1.NodeAllocatableResourceClaimStatus
expectedResource fwk.PodResource
initContainers []v1.Container
overhead *v1.ResourceList
}{
{
name: "requestless container",
@@ -1711,17 +1754,316 @@ func TestPodInfoCalculateResources(t *testing.T) {
Non0Mem: schedutil.DefaultMemoryRequest,
},
},
{
name: "DRA gate disabled, with node allocatable resource claim",
nodeAllocatableResourcesDRAEnabled: false,
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem500M,
},
// We do not set Pod.Spec.ResourceClaims in this test.
// We assume the name maps to Pod.Spec.ResourceClaims[].ResourceClaimName.
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim",
},
},
},
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
},
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu500m.MilliValue(),
Memory: mem500M.Value(),
},
Non0CPU: cpu500m.MilliValue(),
Non0Mem: mem500M.Value(),
},
},
{
name: "container with DRA node allocatable resource claim",
nodeAllocatableResourcesDRAEnabled: true,
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem500M,
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim",
},
},
},
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
},
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu500m.MilliValue() + cpu100m.MilliValue(),
Memory: mem500M.Value() + mem200M.Value(),
},
Non0CPU: cpu500m.MilliValue() + cpu100m.MilliValue(),
Non0Mem: mem500M.Value() + mem200M.Value(),
},
},
{
name: "Multiple DRA node allocatable resource claims",
nodeAllocatableResourcesDRAEnabled: true,
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem500M,
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim-1",
},
{
Name: "node-allocatable-claim-2",
},
},
},
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
},
{
ResourceClaimName: "node-allocatable-claim-2",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
},
},
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu500m.MilliValue() + cpu100m.MilliValue() + cpu100m.MilliValue(),
Memory: mem500M.Value() + mem200M.Value(),
},
Non0CPU: cpu500m.MilliValue() + cpu100m.MilliValue() + cpu100m.MilliValue(),
Non0Mem: mem500M.Value() + mem200M.Value(),
},
},
{
name: "Single DRA claim with multiple resources",
nodeAllocatableResourcesDRAEnabled: true,
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem500M,
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim-1",
},
},
},
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu1000m,
v1.ResourceMemory: mem200M,
},
},
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu500m.MilliValue() + cpu1000m.MilliValue(),
Memory: mem500M.Value() + mem200M.Value(),
},
Non0CPU: cpu500m.MilliValue() + cpu1000m.MilliValue(),
Non0Mem: mem500M.Value() + mem200M.Value(),
},
},
{
name: "DRA node allocatable Resources with Init Container",
nodeAllocatableResourcesDRAEnabled: true,
initContainers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu1200m, // Higher than app container
v1.ResourceMemory: mem500M,
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim-1",
},
},
},
},
},
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem1200M, // Higher than init container
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim-1",
},
},
},
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
},
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu1200m.MilliValue() + cpu100m.MilliValue(), // max(cpu500m, cpu1200m) + draCpu
Memory: mem1200M.Value() + mem200M.Value(), // max(mem500M, mem1200M) + draMem
},
Non0CPU: cpu1200m.MilliValue() + cpu100m.MilliValue(),
Non0Mem: mem1200M.Value() + mem200M.Value(),
},
},
{
name: "DRA node allocatable Resources with Pod Level Resources",
nodeAllocatableResourcesDRAEnabled: true,
podLevelResourcesEnabled: true,
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem500M,
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim-1",
},
},
},
},
},
podResources: &v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu700m,
v1.ResourceMemory: mem800M,
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
},
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu700m.MilliValue(), // pod level requests determines the overall footprint
Memory: mem800M.Value(), // pod level requests determines the overall footprint
},
Non0CPU: cpu700m.MilliValue(),
Non0Mem: mem800M.Value(),
},
},
{
name: "DRA node allocatable Resources with Pod Overhead",
nodeAllocatableResourcesDRAEnabled: true,
containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: cpu500m,
v1.ResourceMemory: mem500M,
},
Claims: []v1.ResourceClaim{
{
Name: "node-allocatable-claim-1",
},
},
},
},
},
nodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
Resources: map[v1.ResourceName]resource.Quantity{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
},
},
// Pod Overhead
overhead: &v1.ResourceList{
v1.ResourceCPU: cpu100m,
v1.ResourceMemory: mem200M,
},
expectedResource: fwk.PodResource{
Resource: &Resource{
MilliCPU: cpu500m.MilliValue() + cpu100m.MilliValue() + cpu100m.MilliValue(), // container + dra + overhead
Memory: mem500M.Value() + mem200M.Value() + mem200M.Value(), // container + dra + overhead
},
Non0CPU: cpu500m.MilliValue() + cpu100m.MilliValue() + cpu100m.MilliValue(),
Non0Mem: mem500M.Value() + mem200M.Value() + mem200M.Value(),
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, tc.podLevelResourcesEnabled)
featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{
features.PodLevelResources: tc.podLevelResourcesEnabled,
features.DRANodeAllocatableResources: tc.nodeAllocatableResourcesDRAEnabled,
})
podSpec := v1.PodSpec{
Resources: tc.podResources,
Containers: tc.containers,
InitContainers: tc.initContainers,
}
if tc.overhead != nil {
podSpec.Overhead = *tc.overhead
}
podInfo := PodInfo{
Pod: &v1.Pod{
Spec: v1.PodSpec{
Resources: tc.podResources,
Containers: tc.containers,
InitContainers: tc.initContainers,
Spec: podSpec,
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: tc.nodeAllocatableResourceClaimStatuses,
},
},
}
@@ -2476,3 +2818,178 @@ func TestQueuedPodInfo_UpdateInvalidatesSignature(t *testing.T) {
t.Errorf("Expected signature to be nil after Update, got '%s'", string(queuedPodInfo.PodSignature))
}
}
func TestUpdateNodeAllocatableDRAClaimState(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRANodeAllocatableResources, true)
claim1NamespacedName := types.NamespacedName{
Name: "node-allocatable-claim-1",
Namespace: "default",
}
claim2NamespacedName := types.NamespacedName{
Name: "node-allocatable-claim-2",
Namespace: "default",
}
tests := []struct {
name string
initialState map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState
pod *v1.Pod
sign int64
expectedState map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState
}{
{
name: "Add pod with single claim",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod1-uid", Namespace: "default"},
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{ResourceClaimName: "node-allocatable-claim-1"},
},
},
},
sign: 1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
},
{
name: "Add pod with multiple claims",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod1-uid", Namespace: "default"},
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
},
{
ResourceClaimName: "node-allocatable-claim-2",
},
},
},
},
sign: 1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
claim2NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
},
{
name: "Add multiple pods with the same claim",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod2-uid", Namespace: "default"},
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{ResourceClaimName: "node-allocatable-claim-1"},
},
},
},
sign: 1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid", "pod2-uid")},
},
},
{
name: "Add multiple pods with different claims",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod2-uid", Namespace: "default"},
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-2",
},
},
},
},
sign: 1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
claim2NamespacedName: {ConsumerPods: sets.New[types.UID]("pod2-uid")},
},
},
{
name: "Remove pod with single claim",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod1-uid", Namespace: "default"},
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{
ResourceClaimName: "node-allocatable-claim-1",
},
},
},
},
sign: -1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{},
},
{
name: "Remove pod with shared claim",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid", "pod2-uid")},
claim2NamespacedName: {ConsumerPods: sets.New[types.UID]("pod2-uid")},
},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod2-uid", Namespace: "default"},
Status: v1.PodStatus{
NodeAllocatableResourceClaimStatuses: []v1.NodeAllocatableResourceClaimStatus{
{ResourceClaimName: "node-allocatable-claim-1"},
{ResourceClaimName: "node-allocatable-claim-2"},
},
},
},
sign: -1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
},
{
name: "Add pod with no claims",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod2-uid", Namespace: "default"},
},
sign: 1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
},
{
name: "Remove pod with no claims",
initialState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
pod: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "pod2-uid", Namespace: "default"},
},
sign: -1,
expectedState: map[types.NamespacedName]*fwk.NodeAllocatableDRAClaimState{
claim1NamespacedName: {ConsumerPods: sets.New[types.UID]("pod1-uid")},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ni := &NodeInfo{
NodeAllocatableDRAClaimStates: tt.initialState,
}
podInfo, _ := NewPodInfo(tt.pod)
ni.updateNodeAllocatableDRAClaimState(podInfo, tt.sign)
if diff := cmp.Diff(tt.expectedState, ni.NodeAllocatableDRAClaimStates, cmp.AllowUnexported(fwk.NodeAllocatableDRAClaimState{})); diff != "" {
t.Errorf("updateNodeAllocatableDRAClaimState() returned diff (-want +got):\\n%s", diff)
}
})
}
}

19
staging/src/k8s.io/kube-scheduler/framework/types.go
View File

@@ -23,6 +23,7 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
ndf "k8s.io/component-helpers/nodedeclaredfeatures"
"k8s.io/klog/v2"
@@ -287,6 +288,8 @@ type NodeInfo interface {
Snapshot() NodeInfo
// String returns representation of human readable format of this NodeInfo.
String() string
// GetNodeAllocatableDRAClaimState returns the node allocatable DRA claim allocation states on this node.
GetNodeAllocatableDRAClaimState() map[types.NamespacedName]*NodeAllocatableDRAClaimState
// AddPodInfo adds pod information to this NodeInfo.
// Consider using this instead of AddPod if a PodInfo is already computed.
@@ -681,3 +684,19 @@ type PodGroupAssignments struct {
// The pods are guaranteed to also be present in the PodGroupInfo.
ProposedAssignments []ProposedAssignment
}
// NodeAllocatableDRAClaimState holds information about a node allocatable resource DRA claim's allocation on a node.
type NodeAllocatableDRAClaimState struct {
// ConsumerPods is a set of UIDs of pods that are consuming the DRA claim on this node.
ConsumerPods sets.Set[types.UID]
}
// Snapshot returns a copy of NodeAllocatableDRAClaimAllocationState with ConsumerPods cloned.
func (s *NodeAllocatableDRAClaimState) Snapshot() *NodeAllocatableDRAClaimState {
if s == nil {
return nil
}
return &NodeAllocatableDRAClaimState{
ConsumerPods: s.ConsumerPods.Clone(),
}
}