mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-28 22:17:14 +00:00
Update semantics to set Preferred field in TopologyHint generation
We now only set Preferred to true if resources can be allocated with a size equal to the minimimum _possible_ mask when all resources are available.
This commit is contained in:
parent
e0e8b3e4fd
commit
eb0216e54e
@ -71,14 +71,26 @@ func (m *manager) GetTopologyHints(pod v1.Pod, container v1.Container) map[strin
|
|||||||
// bits set as the narrowest matching NUMANodeAffinity with 'Preferred: true', and
|
// bits set as the narrowest matching NUMANodeAffinity with 'Preferred: true', and
|
||||||
// marking all others with 'Preferred: false'.
|
// marking all others with 'Preferred: false'.
|
||||||
func (m *manager) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, request int) []topologymanager.TopologyHint {
|
func (m *manager) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, request int) []topologymanager.TopologyHint {
|
||||||
// Initialize minAffinity to a full affinity mask.
|
// Initialize minAffinitySize to include all NUMA Nodes.
|
||||||
minAffinity, _ := socketmask.NewSocketMask()
|
minAffinitySize := m.topology.CPUDetails.NUMANodes().Size()
|
||||||
minAffinity.Fill()
|
// Initialize minSocketsOnMinAffinity to include all Sockets.
|
||||||
|
minSocketsOnMinAffinity := m.topology.CPUDetails.Sockets().Size()
|
||||||
|
|
||||||
// Iterate through all combinations of socketMasks and build hints from them.
|
// Iterate through all combinations of socketMasks and build hints from them.
|
||||||
hints := []topologymanager.TopologyHint{}
|
hints := []topologymanager.TopologyHint{}
|
||||||
socketmask.IterateSocketMasks(m.topology.CPUDetails.NUMANodes().ToSlice(), func(mask socketmask.SocketMask) {
|
socketmask.IterateSocketMasks(m.topology.CPUDetails.NUMANodes().ToSlice(), func(mask socketmask.SocketMask) {
|
||||||
// Check to see if we have enough CPUs available on the current
|
// First, update minAffinitySize and minSocketsOnMinAffinity for the
|
||||||
|
// current request size.
|
||||||
|
cpusInMask := m.topology.CPUDetails.CPUsInNUMANodes(mask.GetSockets()...).Size()
|
||||||
|
socketsInMask := m.topology.CPUDetails.SocketsInNUMANodes(mask.GetSockets()...).Size()
|
||||||
|
if cpusInMask >= request && mask.Count() < minAffinitySize {
|
||||||
|
minAffinitySize = mask.Count()
|
||||||
|
if socketsInMask < minSocketsOnMinAffinity {
|
||||||
|
minSocketsOnMinAffinity = socketsInMask
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then check to see if we have enough CPUs available on the current
|
||||||
// SocketMask to satisfy the CPU request.
|
// SocketMask to satisfy the CPU request.
|
||||||
numMatching := 0
|
numMatching := 0
|
||||||
for _, c := range availableCPUs.ToSlice() {
|
for _, c := range availableCPUs.ToSlice() {
|
||||||
@ -99,20 +111,19 @@ func (m *manager) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, request
|
|||||||
NUMANodeAffinity: mask,
|
NUMANodeAffinity: mask,
|
||||||
Preferred: false,
|
Preferred: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Update minAffinity if relevant
|
|
||||||
if mask.IsNarrowerThan(minAffinity) {
|
|
||||||
minAffinity = mask
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Loop back through all hints and update the 'Preferred' field based on
|
// Loop back through all hints and update the 'Preferred' field based on
|
||||||
// counting the number of bits sets in the affinity mask and comparing it
|
// counting the number of bits sets in the affinity mask and comparing it
|
||||||
// to the minAffinity. Only those with an equal number of bits set will be
|
// to the minAffinitySize. Only those with an equal number of bits set (and
|
||||||
// considered preferred.
|
// with a minimal set of sockets) will be considered preferred.
|
||||||
for i := range hints {
|
for i := range hints {
|
||||||
if hints[i].NUMANodeAffinity.Count() == minAffinity.Count() {
|
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
|
||||||
hints[i].Preferred = true
|
nodes := hints[i].NUMANodeAffinity.GetSockets()
|
||||||
|
numSockets := m.topology.CPUDetails.SocketsInNUMANodes(nodes...).Size()
|
||||||
|
if numSockets == minSocketsOnMinAffinity {
|
||||||
|
hints[i].Preferred = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,28 +75,18 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11),
|
1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11),
|
||||||
}
|
}
|
||||||
|
|
||||||
topology, _ := topology.Discover(&machineInfo, numaNodeInfo)
|
|
||||||
|
|
||||||
m := manager{
|
|
||||||
policy: &staticPolicy{
|
|
||||||
topology: topology,
|
|
||||||
},
|
|
||||||
state: &mockState{
|
|
||||||
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
|
|
||||||
},
|
|
||||||
topology: topology,
|
|
||||||
}
|
|
||||||
|
|
||||||
tcases := []struct {
|
tcases := []struct {
|
||||||
name string
|
name string
|
||||||
pod v1.Pod
|
pod v1.Pod
|
||||||
container v1.Container
|
container v1.Container
|
||||||
|
defaultCPUSet cpuset.CPUSet
|
||||||
expectedHints []topologymanager.TopologyHint
|
expectedHints []topologymanager.TopologyHint
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Request 2 CPUs; 4 available on Socket 0, 6 available on Socket 1",
|
name: "Request 2 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
|
||||||
pod: *testPod1,
|
pod: *testPod1,
|
||||||
container: *testContainer1,
|
container: *testContainer1,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
|
||||||
expectedHints: []topologymanager.TopologyHint{
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
{
|
{
|
||||||
NUMANodeAffinity: firstSocketMask,
|
NUMANodeAffinity: firstSocketMask,
|
||||||
@ -113,9 +103,10 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Request 5 CPUs; 4 available on Socket 0, 6 available on Socket 1",
|
name: "Request 5 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
|
||||||
pod: *testPod2,
|
pod: *testPod2,
|
||||||
container: *testContainer2,
|
container: *testContainer2,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
|
||||||
expectedHints: []topologymanager.TopologyHint{
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
{
|
{
|
||||||
NUMANodeAffinity: secondSocketMask,
|
NUMANodeAffinity: secondSocketMask,
|
||||||
@ -128,9 +119,10 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Request 7 CPUs, 4 available on Socket 0, 6 available on Socket 1",
|
name: "Request 7 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
|
||||||
pod: *testPod3,
|
pod: *testPod3,
|
||||||
container: *testContainer3,
|
container: *testContainer3,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
|
||||||
expectedHints: []topologymanager.TopologyHint{
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
{
|
{
|
||||||
NUMANodeAffinity: crossSocketMask,
|
NUMANodeAffinity: crossSocketMask,
|
||||||
@ -139,13 +131,38 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Request 11 CPUs, 4 available on Socket 0, 6 available on Socket 1",
|
name: "Request 11 CPUs, 4 available on NUMA 0, 6 available on NUMA 1",
|
||||||
pod: *testPod4,
|
pod: *testPod4,
|
||||||
container: *testContainer4,
|
container: *testContainer4,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(2, 3, 4, 5, 6, 7, 8, 9, 10, 11),
|
||||||
expectedHints: nil,
|
expectedHints: nil,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "Request 2 CPUs, 1 available on NUMA 0, 1 available on NUMA 1",
|
||||||
|
pod: *testPod1,
|
||||||
|
container: *testContainer1,
|
||||||
|
defaultCPUSet: cpuset.NewCPUSet(0, 3),
|
||||||
|
expectedHints: []topologymanager.TopologyHint{
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: crossSocketMask,
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, tc := range tcases {
|
for _, tc := range tcases {
|
||||||
|
topology, _ := topology.Discover(&machineInfo, numaNodeInfo)
|
||||||
|
|
||||||
|
m := manager{
|
||||||
|
policy: &staticPolicy{
|
||||||
|
topology: topology,
|
||||||
|
},
|
||||||
|
state: &mockState{
|
||||||
|
defaultCPUSet: tc.defaultCPUSet,
|
||||||
|
},
|
||||||
|
topology: topology,
|
||||||
|
}
|
||||||
|
|
||||||
hints := m.GetTopologyHints(tc.pod, tc.container)[string(v1.ResourceCPU)]
|
hints := m.GetTopologyHints(tc.pod, tc.container)[string(v1.ResourceCPU)]
|
||||||
if len(tc.expectedHints) == 0 && len(hints) == 0 {
|
if len(tc.expectedHints) == 0 && len(hints) == 0 {
|
||||||
continue
|
continue
|
||||||
|
@ -73,13 +73,30 @@ func (m *ManagerImpl) getAvailableDevices(resource string) sets.String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.String, request int) []topologymanager.TopologyHint {
|
func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.String, request int) []topologymanager.TopologyHint {
|
||||||
// Initialize minAffinity to a full affinity mask.
|
// Initialize minAffinitySize to include all NUMA Nodes
|
||||||
minAffinity, _ := socketmask.NewSocketMask(m.numaNodes...)
|
minAffinitySize := len(m.numaNodes)
|
||||||
|
|
||||||
// Iterate through all combinations of NUMA Nodes and build hints from them.
|
// Iterate through all combinations of NUMA Nodes and build hints from them.
|
||||||
hints := []topologymanager.TopologyHint{}
|
hints := []topologymanager.TopologyHint{}
|
||||||
socketmask.IterateSocketMasks(m.numaNodes, func(mask socketmask.SocketMask) {
|
socketmask.IterateSocketMasks(m.numaNodes, func(mask socketmask.SocketMask) {
|
||||||
// Check to see if we have enough devices available on the current
|
// First, update minAffinitySize for the current request size.
|
||||||
|
devicesInMask := 0
|
||||||
|
for _, device := range m.allDevices[resource] {
|
||||||
|
if device.Topology == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, node := range device.Topology.Nodes {
|
||||||
|
if mask.IsSet(int(node.ID)) {
|
||||||
|
devicesInMask++
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if devicesInMask >= request && mask.Count() < minAffinitySize {
|
||||||
|
minAffinitySize = mask.Count()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then check to see if we have enough devices available on the current
|
||||||
// NUMA Node combination to satisfy the device request.
|
// NUMA Node combination to satisfy the device request.
|
||||||
numMatching := 0
|
numMatching := 0
|
||||||
for d := range devices {
|
for d := range devices {
|
||||||
@ -106,11 +123,6 @@ func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.
|
|||||||
NUMANodeAffinity: mask,
|
NUMANodeAffinity: mask,
|
||||||
Preferred: false,
|
Preferred: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Update minAffinity if relevant
|
|
||||||
if mask.IsNarrowerThan(minAffinity) {
|
|
||||||
minAffinity = mask
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Loop back through all hints and update the 'Preferred' field based on
|
// Loop back through all hints and update the 'Preferred' field based on
|
||||||
@ -118,7 +130,7 @@ func (m *ManagerImpl) generateDeviceTopologyHints(resource string, devices sets.
|
|||||||
// to the minAffinity. Only those with an equal number of bits set will be
|
// to the minAffinity. Only those with an equal number of bits set will be
|
||||||
// considered preferred.
|
// considered preferred.
|
||||||
for i := range hints {
|
for i := range hints {
|
||||||
if hints[i].NUMANodeAffinity.Count() == minAffinity.Count() {
|
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
|
||||||
hints[i].Preferred = true
|
hints[i].Preferred = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -58,10 +58,11 @@ func makeSocketMask(sockets ...int) socketmask.SocketMask {
|
|||||||
|
|
||||||
func TestGetTopologyHints(t *testing.T) {
|
func TestGetTopologyHints(t *testing.T) {
|
||||||
tcases := []struct {
|
tcases := []struct {
|
||||||
description string
|
description string
|
||||||
request map[string]string
|
request map[string]string
|
||||||
devices map[string][]pluginapi.Device
|
devices map[string][]pluginapi.Device
|
||||||
expectedHints map[string][]topologymanager.TopologyHint
|
allocatedDevices map[string][]string
|
||||||
|
expectedHints map[string][]topologymanager.TopologyHint
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
description: "Single Request, no alignment",
|
description: "Single Request, no alignment",
|
||||||
@ -180,6 +181,31 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
description: "Request for 2, optimal on 1 NUMA node, forced cross-NUMA",
|
||||||
|
request: map[string]string{
|
||||||
|
"testdevice": "2",
|
||||||
|
},
|
||||||
|
devices: map[string][]pluginapi.Device{
|
||||||
|
"testdevice": {
|
||||||
|
makeNUMADevice("Dev1", 0),
|
||||||
|
makeNUMADevice("Dev2", 1),
|
||||||
|
makeNUMADevice("Dev3", 0),
|
||||||
|
makeNUMADevice("Dev4", 1),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
allocatedDevices: map[string][]string{
|
||||||
|
"testdevice": {"Dev1", "Dev2"},
|
||||||
|
},
|
||||||
|
expectedHints: map[string][]topologymanager.TopologyHint{
|
||||||
|
"testdevice": {
|
||||||
|
{
|
||||||
|
NUMANodeAffinity: makeSocketMask(0, 1),
|
||||||
|
Preferred: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
description: "2 device types, mixed configuration",
|
description: "2 device types, mixed configuration",
|
||||||
request: map[string]string{
|
request: map[string]string{
|
||||||
@ -254,6 +280,14 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for r := range tc.allocatedDevices {
|
||||||
|
m.allocatedDevices[r] = sets.NewString()
|
||||||
|
|
||||||
|
for _, d := range tc.allocatedDevices[r] {
|
||||||
|
m.allocatedDevices[r].Insert(d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
hints := m.GetTopologyHints(*pod, pod.Spec.Containers[0])
|
hints := m.GetTopologyHints(*pod, pod.Spec.Containers[0])
|
||||||
|
|
||||||
for r := range tc.expectedHints {
|
for r := range tc.expectedHints {
|
||||||
@ -276,6 +310,7 @@ func TestTopologyAlignedAllocation(t *testing.T) {
|
|||||||
resource string
|
resource string
|
||||||
request int
|
request int
|
||||||
devices []pluginapi.Device
|
devices []pluginapi.Device
|
||||||
|
allocatedDevices []string
|
||||||
hint topologymanager.TopologyHint
|
hint topologymanager.TopologyHint
|
||||||
expectedAllocation int
|
expectedAllocation int
|
||||||
expectedAlignment map[int]int
|
expectedAlignment map[int]int
|
||||||
|
Loading…
Reference in New Issue
Block a user