diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 21d5530481a..50011eae887 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -26,20 +26,133 @@ import ( "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) +type numaOrSocketsFirstFuncs interface { + takeFullFirstLevel() + takeFullSecondLevel() + sortAvailableNUMANodes() []int + sortAvailableSockets() []int + sortAvailableCores() []int +} + +type numaFirst struct{ acc *cpuAccumulator } +type socketsFirst struct{ acc *cpuAccumulator } + +var _ numaOrSocketsFirstFuncs = (*numaFirst)(nil) +var _ numaOrSocketsFirstFuncs = (*socketsFirst)(nil) + +// If NUMA nodes are higher in the memory hierarchy than sockets, then we take +// from the set of NUMA Nodes as the first level. +func (n *numaFirst) takeFullFirstLevel() { + n.acc.takeFullNUMANodes() +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, then we take +// from the set of sockets as the second level. +func (n *numaFirst) takeFullSecondLevel() { + n.acc.takeFullSockets() +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, then just +// sort the NUMA nodes directly, and return them. +func (n *numaFirst) sortAvailableNUMANodes() []int { + numas := n.acc.details.NUMANodes().ToSliceNoSort() + n.acc.sort(numas, n.acc.details.CPUsInNUMANodes) + return numas +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, then we need +// to pull the set of sockets out of each sorted NUMA node, and accumulate the +// partial order across them. +func (n *numaFirst) sortAvailableSockets() []int { + var result []int + for _, numa := range n.sortAvailableNUMANodes() { + sockets := n.acc.details.SocketsInNUMANodes(numa).ToSliceNoSort() + n.acc.sort(sockets, n.acc.details.CPUsInSockets) + result = append(result, sockets...) + } + return result +} + +// If NUMA nodes are higher in the memory hierarchy than sockets, then +// cores sit directly below sockets in the memory hierarchy. +func (n *numaFirst) sortAvailableCores() []int { + var result []int + for _, socket := range n.acc.sortAvailableSockets() { + cores := n.acc.details.CoresInSockets(socket).ToSliceNoSort() + n.acc.sort(cores, n.acc.details.CPUsInCores) + result = append(result, cores...) + } + return result +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then we take +// from the set of sockets as the first level. +func (s *socketsFirst) takeFullFirstLevel() { + s.acc.takeFullSockets() +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then we take +// from the set of NUMA Nodes as the second level. +func (s *socketsFirst) takeFullSecondLevel() { + s.acc.takeFullNUMANodes() +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then we need +// to pull the set of NUMA nodes out of each sorted Socket, and accumulate the +// partial order across them. +func (s *socketsFirst) sortAvailableNUMANodes() []int { + var result []int + for _, socket := range s.sortAvailableSockets() { + numas := s.acc.details.NUMANodesInSockets(socket).ToSliceNoSort() + s.acc.sort(numas, s.acc.details.CPUsInNUMANodes) + result = append(result, numas...) + } + return result +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then just +// sort the sockets directly, and return them. +func (s *socketsFirst) sortAvailableSockets() []int { + sockets := s.acc.details.Sockets().ToSliceNoSort() + s.acc.sort(sockets, s.acc.details.CPUsInSockets) + return sockets +} + +// If sockets are higher in the memory hierarchy than NUMA nodes, then cores +// sit directly below NUMA Nodes in the memory hierarchy. +func (s *socketsFirst) sortAvailableCores() []int { + var result []int + for _, numa := range s.acc.sortAvailableNUMANodes() { + cores := s.acc.details.CoresInNUMANodes(numa).ToSliceNoSort() + s.acc.sort(cores, s.acc.details.CPUsInCores) + result = append(result, cores...) + } + return result +} + type cpuAccumulator struct { - topo *topology.CPUTopology - details topology.CPUDetails - numCPUsNeeded int - result cpuset.CPUSet + topo *topology.CPUTopology + details topology.CPUDetails + numCPUsNeeded int + result cpuset.CPUSet + numaOrSocketsFirst numaOrSocketsFirstFuncs } func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) *cpuAccumulator { - return &cpuAccumulator{ + acc := &cpuAccumulator{ topo: topo, details: topo.CPUDetails.KeepOnly(availableCPUs), numCPUsNeeded: numCPUs, result: cpuset.NewCPUSet(), } + + if topo.NumSockets >= topo.NumNUMANodes { + acc.numaOrSocketsFirst = &numaFirst{acc} + } else { + acc.numaOrSocketsFirst = &socketsFirst{acc} + } + + return acc } // Returns true if the supplied NUMANode is fully available in `topoDetails`. @@ -116,78 +229,19 @@ func (a *cpuAccumulator) sort(ids []int, getCPUs func(ids ...int) cpuset.CPUSet) }) } -// Sort all NUMA nodes with free CPUs: -// - If NUMA nodes are higher than sockets in the memory hierarchy then sort -// them directly using the sort() algorithm defined above. -// - Otherwise sort them: -// - First by socket using sortAvailableSockets(). -// - Then within each socket, using the sort() algorithm defined above. +// Sort all NUMA nodes with free CPUs. func (a *cpuAccumulator) sortAvailableNUMANodes() []int { - // If NUMA nodes are equal or higher in the memory hierarchy than sockets - if a.topo.NumSockets >= a.topo.NumNUMANodes { - numas := a.details.NUMANodes().ToSliceNoSort() - a.sort(numas, a.details.CPUsInNUMANodes) - return numas - } - - // Otherwise each socket has multiple NUMA nodes - var result []int - for _, socket := range a.sortAvailableSockets() { - numas := a.details.NUMANodesInSockets(socket).ToSliceNoSort() - a.sort(numas, a.details.CPUsInNUMANodes) - result = append(result, numas...) - } - return result + return a.numaOrSocketsFirst.sortAvailableNUMANodes() } -// Sort all sockets with free CPUs: -// - If sockets are higher than NUMA nodes in the memory hierarchy then sort -// them directly using the sort() algorithm defined above. -// - Otherwise sort them: -// - First by NUMA node using sortAvailableNUMANodes(). -// - Then within each NUMA node, using the sort() algorithm defined above. +// Sort all sockets with free CPUs. func (a *cpuAccumulator) sortAvailableSockets() []int { - // If sockets are higher than NUMA nodes in the memory hierarchy - if a.topo.NumNUMANodes >= a.topo.NumSockets { - sockets := a.details.Sockets().ToSliceNoSort() - a.sort(sockets, a.details.CPUsInSockets) - return sockets - } - - // Otherwise each NUMA Node has multiple sockets - var result []int - for _, numa := range a.sortAvailableNUMANodes() { - sockets := a.details.SocketsInNUMANodes(numa).ToSliceNoSort() - a.sort(sockets, a.details.CPUsInSockets) - result = append(result, sockets...) - } - return result + return a.numaOrSocketsFirst.sortAvailableSockets() } // Sort all cores with free CPUs: -// - First by socket (or NUMA node) using sortAvailableSockets() (or sortAvailableNUMANodes()). -// - Then within each socket or NUMA node, using the sort() algorithm defined above. func (a *cpuAccumulator) sortAvailableCores() []int { - // If NUMA nodes are higher in the memory hierarchy than sockets, then - // cores sit directly below sockets in the memory hierarchy. - if a.topo.NumSockets >= a.topo.NumNUMANodes { - var result []int - for _, socket := range a.sortAvailableSockets() { - cores := a.details.CoresInSockets(socket).ToSliceNoSort() - a.sort(cores, a.details.CPUsInCores) - result = append(result, cores...) - } - return result - } - - // Otherwise they sit directly below NUMA nodes. - var result []int - for _, numa := range a.sortAvailableNUMANodes() { - cores := a.details.CoresInNUMANodes(numa).ToSliceNoSort() - a.sort(cores, a.details.CPUsInCores) - result = append(result, cores...) - } - return result + return a.numaOrSocketsFirst.sortAvailableCores() } // Sort all available CPUs: @@ -278,26 +332,13 @@ func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, num // requires at least a NUMA node or socket's-worth of CPUs. If NUMA // Nodes map to 1 or more sockets, pull from NUMA nodes first. // Otherwise pull from sockets first. - if acc.topo.NumSockets >= acc.topo.NumNUMANodes { - acc.takeFullNUMANodes() - if acc.isSatisfied() { - return acc.result, nil - } - - acc.takeFullSockets() - if acc.isSatisfied() { - return acc.result, nil - } - } else { - acc.takeFullSockets() - if acc.isSatisfied() { - return acc.result, nil - } - - acc.takeFullNUMANodes() - if acc.isSatisfied() { - return acc.result, nil - } + acc.numaOrSocketsFirst.takeFullFirstLevel() + if acc.isSatisfied() { + return acc.result, nil + } + acc.numaOrSocketsFirst.takeFullSecondLevel() + if acc.isSatisfied() { + return acc.result, nil } // 2. Acquire whole cores, if available and the container requires at least