mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-31 07:20:13 +00:00
Abstract out whether NUMA or Sockets come first in the memory hierarchy
This allows us to get rid of the check for determining which one is higher all throughout the code. Now we just check once and instantiate an interface of the appropriate type that makes sure the ordering in the hierarchy is preserved through the appropriate calls. Signed-off-by: Kevin Klues <kklues@nvidia.com>
This commit is contained in:
Kevin Klues
parent
17c7e86c6d
commit
aff54a0914
@ -26,20 +26,133 @@ import (
|
||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
||||
)
|
||||
|
||||
type numaOrSocketsFirstFuncs interface {
|
||||
takeFullFirstLevel()
|
||||
takeFullSecondLevel()
|
||||
sortAvailableNUMANodes() []int
|
||||
sortAvailableSockets() []int
|
||||
sortAvailableCores() []int
|
||||
}
|
||||
|
||||
type numaFirst struct{ acc *cpuAccumulator }
|
||||
type socketsFirst struct{ acc *cpuAccumulator }
|
||||
|
||||
var _ numaOrSocketsFirstFuncs = (*numaFirst)(nil)
|
||||
var _ numaOrSocketsFirstFuncs = (*socketsFirst)(nil)
|
||||
|
||||
// If NUMA nodes are higher in the memory hierarchy than sockets, then we take
|
||||
// from the set of NUMA Nodes as the first level.
|
||||
func (n *numaFirst) takeFullFirstLevel() {
|
||||
n.acc.takeFullNUMANodes()
|
||||
}
|
||||
|
||||
// If NUMA nodes are higher in the memory hierarchy than sockets, then we take
|
||||
// from the set of sockets as the second level.
|
||||
func (n *numaFirst) takeFullSecondLevel() {
|
||||
n.acc.takeFullSockets()
|
||||
}
|
||||
|
||||
// If NUMA nodes are higher in the memory hierarchy than sockets, then just
|
||||
// sort the NUMA nodes directly, and return them.
|
||||
func (n *numaFirst) sortAvailableNUMANodes() []int {
|
||||
numas := n.acc.details.NUMANodes().ToSliceNoSort()
|
||||
n.acc.sort(numas, n.acc.details.CPUsInNUMANodes)
|
||||
return numas
|
||||
}
|
||||
|
||||
// If NUMA nodes are higher in the memory hierarchy than sockets, then we need
|
||||
// to pull the set of sockets out of each sorted NUMA node, and accumulate the
|
||||
// partial order across them.
|
||||
func (n *numaFirst) sortAvailableSockets() []int {
|
||||
var result []int
|
||||
for _, numa := range n.sortAvailableNUMANodes() {
|
||||
sockets := n.acc.details.SocketsInNUMANodes(numa).ToSliceNoSort()
|
||||
n.acc.sort(sockets, n.acc.details.CPUsInSockets)
|
||||
result = append(result, sockets...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// If NUMA nodes are higher in the memory hierarchy than sockets, then
|
||||
// cores sit directly below sockets in the memory hierarchy.
|
||||
func (n *numaFirst) sortAvailableCores() []int {
|
||||
var result []int
|
||||
for _, socket := range n.acc.sortAvailableSockets() {
|
||||
cores := n.acc.details.CoresInSockets(socket).ToSliceNoSort()
|
||||
n.acc.sort(cores, n.acc.details.CPUsInCores)
|
||||
result = append(result, cores...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// If sockets are higher in the memory hierarchy than NUMA nodes, then we take
|
||||
// from the set of sockets as the first level.
|
||||
func (s *socketsFirst) takeFullFirstLevel() {
|
||||
s.acc.takeFullSockets()
|
||||
}
|
||||
|
||||
// If sockets are higher in the memory hierarchy than NUMA nodes, then we take
|
||||
// from the set of NUMA Nodes as the second level.
|
||||
func (s *socketsFirst) takeFullSecondLevel() {
|
||||
s.acc.takeFullNUMANodes()
|
||||
}
|
||||
|
||||
// If sockets are higher in the memory hierarchy than NUMA nodes, then we need
|
||||
// to pull the set of NUMA nodes out of each sorted Socket, and accumulate the
|
||||
// partial order across them.
|
||||
func (s *socketsFirst) sortAvailableNUMANodes() []int {
|
||||
var result []int
|
||||
for _, socket := range s.sortAvailableSockets() {
|
||||
numas := s.acc.details.NUMANodesInSockets(socket).ToSliceNoSort()
|
||||
s.acc.sort(numas, s.acc.details.CPUsInNUMANodes)
|
||||
result = append(result, numas...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// If sockets are higher in the memory hierarchy than NUMA nodes, then just
|
||||
// sort the sockets directly, and return them.
|
||||
func (s *socketsFirst) sortAvailableSockets() []int {
|
||||
sockets := s.acc.details.Sockets().ToSliceNoSort()
|
||||
s.acc.sort(sockets, s.acc.details.CPUsInSockets)
|
||||
return sockets
|
||||
}
|
||||
|
||||
// If sockets are higher in the memory hierarchy than NUMA nodes, then cores
|
||||
// sit directly below NUMA Nodes in the memory hierarchy.
|
||||
func (s *socketsFirst) sortAvailableCores() []int {
|
||||
var result []int
|
||||
for _, numa := range s.acc.sortAvailableNUMANodes() {
|
||||
cores := s.acc.details.CoresInNUMANodes(numa).ToSliceNoSort()
|
||||
s.acc.sort(cores, s.acc.details.CPUsInCores)
|
||||
result = append(result, cores...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type cpuAccumulator struct {
|
||||
topo *topology.CPUTopology
|
||||
details topology.CPUDetails
|
||||
numCPUsNeeded int
|
||||
result cpuset.CPUSet
|
||||
topo *topology.CPUTopology
|
||||
details topology.CPUDetails
|
||||
numCPUsNeeded int
|
||||
result cpuset.CPUSet
|
||||
numaOrSocketsFirst numaOrSocketsFirstFuncs
|
||||
}
|
||||
|
||||
func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) *cpuAccumulator {
|
||||
return &cpuAccumulator{
|
||||
acc := &cpuAccumulator{
|
||||
topo: topo,
|
||||
details: topo.CPUDetails.KeepOnly(availableCPUs),
|
||||
numCPUsNeeded: numCPUs,
|
||||
result: cpuset.NewCPUSet(),
|
||||
}
|
||||
|
||||
if topo.NumSockets >= topo.NumNUMANodes {
|
||||
acc.numaOrSocketsFirst = &numaFirst{acc}
|
||||
} else {
|
||||
acc.numaOrSocketsFirst = &socketsFirst{acc}
|
||||
}
|
||||
|
||||
return acc
|
||||
}
|
||||
|
||||
// Returns true if the supplied NUMANode is fully available in `topoDetails`.
|
||||
@ -116,78 +229,19 @@ func (a *cpuAccumulator) sort(ids []int, getCPUs func(ids ...int) cpuset.CPUSet)
|
||||
})
|
||||
}
|
||||
|
||||
// Sort all NUMA nodes with free CPUs:
|
||||
// - If NUMA nodes are higher than sockets in the memory hierarchy then sort
|
||||
// them directly using the sort() algorithm defined above.
|
||||
// - Otherwise sort them:
|
||||
// - First by socket using sortAvailableSockets().
|
||||
// - Then within each socket, using the sort() algorithm defined above.
|
||||
// Sort all NUMA nodes with free CPUs.
|
||||
func (a *cpuAccumulator) sortAvailableNUMANodes() []int {
|
||||
// If NUMA nodes are equal or higher in the memory hierarchy than sockets
|
||||
if a.topo.NumSockets >= a.topo.NumNUMANodes {
|
||||
numas := a.details.NUMANodes().ToSliceNoSort()
|
||||
a.sort(numas, a.details.CPUsInNUMANodes)
|
||||
return numas
|
||||
}
|
||||
|
||||
// Otherwise each socket has multiple NUMA nodes
|
||||
var result []int
|
||||
for _, socket := range a.sortAvailableSockets() {
|
||||
numas := a.details.NUMANodesInSockets(socket).ToSliceNoSort()
|
||||
a.sort(numas, a.details.CPUsInNUMANodes)
|
||||
result = append(result, numas...)
|
||||
}
|
||||
return result
|
||||
return a.numaOrSocketsFirst.sortAvailableNUMANodes()
|
||||
}
|
||||
|
||||
// Sort all sockets with free CPUs:
|
||||
// - If sockets are higher than NUMA nodes in the memory hierarchy then sort
|
||||
// them directly using the sort() algorithm defined above.
|
||||
// - Otherwise sort them:
|
||||
// - First by NUMA node using sortAvailableNUMANodes().
|
||||
// - Then within each NUMA node, using the sort() algorithm defined above.
|
||||
// Sort all sockets with free CPUs.
|
||||
func (a *cpuAccumulator) sortAvailableSockets() []int {
|
||||
// If sockets are higher than NUMA nodes in the memory hierarchy
|
||||
if a.topo.NumNUMANodes >= a.topo.NumSockets {
|
||||
sockets := a.details.Sockets().ToSliceNoSort()
|
||||
a.sort(sockets, a.details.CPUsInSockets)
|
||||
return sockets
|
||||
}
|
||||
|
||||
// Otherwise each NUMA Node has multiple sockets
|
||||
var result []int
|
||||
for _, numa := range a.sortAvailableNUMANodes() {
|
||||
sockets := a.details.SocketsInNUMANodes(numa).ToSliceNoSort()
|
||||
a.sort(sockets, a.details.CPUsInSockets)
|
||||
result = append(result, sockets...)
|
||||
}
|
||||
return result
|
||||
return a.numaOrSocketsFirst.sortAvailableSockets()
|
||||
}
|
||||
|
||||
// Sort all cores with free CPUs:
|
||||
// - First by socket (or NUMA node) using sortAvailableSockets() (or sortAvailableNUMANodes()).
|
||||
// - Then within each socket or NUMA node, using the sort() algorithm defined above.
|
||||
func (a *cpuAccumulator) sortAvailableCores() []int {
|
||||
// If NUMA nodes are higher in the memory hierarchy than sockets, then
|
||||
// cores sit directly below sockets in the memory hierarchy.
|
||||
if a.topo.NumSockets >= a.topo.NumNUMANodes {
|
||||
var result []int
|
||||
for _, socket := range a.sortAvailableSockets() {
|
||||
cores := a.details.CoresInSockets(socket).ToSliceNoSort()
|
||||
a.sort(cores, a.details.CPUsInCores)
|
||||
result = append(result, cores...)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Otherwise they sit directly below NUMA nodes.
|
||||
var result []int
|
||||
for _, numa := range a.sortAvailableNUMANodes() {
|
||||
cores := a.details.CoresInNUMANodes(numa).ToSliceNoSort()
|
||||
a.sort(cores, a.details.CPUsInCores)
|
||||
result = append(result, cores...)
|
||||
}
|
||||
return result
|
||||
return a.numaOrSocketsFirst.sortAvailableCores()
|
||||
}
|
||||
|
||||
// Sort all available CPUs:
|
||||
@ -278,26 +332,13 @@ func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, num
|
||||
// requires at least a NUMA node or socket's-worth of CPUs. If NUMA
|
||||
// Nodes map to 1 or more sockets, pull from NUMA nodes first.
|
||||
// Otherwise pull from sockets first.
|
||||
if acc.topo.NumSockets >= acc.topo.NumNUMANodes {
|
||||
acc.takeFullNUMANodes()
|
||||
if acc.isSatisfied() {
|
||||
return acc.result, nil
|
||||
}
|
||||
|
||||
acc.takeFullSockets()
|
||||
if acc.isSatisfied() {
|
||||
return acc.result, nil
|
||||
}
|
||||
} else {
|
||||
acc.takeFullSockets()
|
||||
if acc.isSatisfied() {
|
||||
return acc.result, nil
|
||||
}
|
||||
|
||||
acc.takeFullNUMANodes()
|
||||
if acc.isSatisfied() {
|
||||
return acc.result, nil
|
||||
}
|
||||
acc.numaOrSocketsFirst.takeFullFirstLevel()
|
||||
if acc.isSatisfied() {
|
||||
return acc.result, nil
|
||||
}
|
||||
acc.numaOrSocketsFirst.takeFullSecondLevel()
|
||||
if acc.isSatisfied() {
|
||||
return acc.result, nil
|
||||
}
|
||||
|
||||
// 2. Acquire whole cores, if available and the container requires at least
|
||||
|
||||
Loading…
Reference in New Issue
Block a user