mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-29 06:27:05 +00:00
Move function from top-level TopologyManager to best-effort policy
This is in preparation for removing the special-case of the SingleNumaNode policy in mergeProvidersHints() in favor of a custom merging strategy with much less overhead.
This commit is contained in:
Adrian Chiris
Kevin Klues
parent
d95464645c
commit
5f7db54d3c
@ -17,6 +17,8 @@ limitations under the License.
|
|||||||
package topologymanager
|
package topologymanager
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"k8s.io/klog"
|
||||||
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
|
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -50,3 +52,148 @@ func (p *bestEffortPolicy) Merge(providersHints []map[string][]TopologyHint) (To
|
|||||||
admit := p.canAdmitPodResult(&hint)
|
admit := p.canAdmitPodResult(&hint)
|
||||||
return hint, admit
|
return hint, admit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'.
|
||||||
|
//
|
||||||
|
// This procedure is implemented as a recursive function over the set of hints
|
||||||
|
// in 'allproviderHints[i]'. It applies the function 'callback' to each
|
||||||
|
// permutation as it is found. It is the equivalent of:
|
||||||
|
//
|
||||||
|
// for i := 0; i < len(providerHints[0]); i++
|
||||||
|
// for j := 0; j < len(providerHints[1]); j++
|
||||||
|
// for k := 0; k < len(providerHints[2]); k++
|
||||||
|
// ...
|
||||||
|
// for z := 0; z < len(providerHints[-1]); z++
|
||||||
|
// permutation := []TopologyHint{
|
||||||
|
// providerHints[0][i],
|
||||||
|
// providerHints[1][j],
|
||||||
|
// providerHints[2][k],
|
||||||
|
// ...
|
||||||
|
// provideryHints[-1][z]
|
||||||
|
// }
|
||||||
|
// callback(permutation)
|
||||||
|
func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) {
|
||||||
|
// Internal helper function to accumulate the permutation before calling the callback.
|
||||||
|
var iterate func(i int, accum []TopologyHint)
|
||||||
|
iterate = func(i int, accum []TopologyHint) {
|
||||||
|
// Base case: we have looped through all providers and have a full permutation.
|
||||||
|
if i == len(allProviderHints) {
|
||||||
|
callback(accum)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Loop through all hints for provider 'i', and recurse to build the
|
||||||
|
// the permutation of this hint with all hints from providers 'i++'.
|
||||||
|
for j := range allProviderHints[i] {
|
||||||
|
iterate(i+1, append(accum, allProviderHints[i][j]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
iterate(0, []TopologyHint{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge the hints from all hint providers to find the best one.
|
||||||
|
func mergeProvidersHints(policy Policy, numaNodes []int, providersHints []map[string][]TopologyHint) TopologyHint {
|
||||||
|
// Set the default affinity as an any-numa affinity containing the list
|
||||||
|
// of NUMA Nodes available on this machine.
|
||||||
|
defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)
|
||||||
|
|
||||||
|
// Loop through all hint providers and save an accumulated list of the
|
||||||
|
// hints returned by each hint provider. If no hints are provided, assume
|
||||||
|
// that provider has no preference for topology-aware allocation.
|
||||||
|
var allProviderHints [][]TopologyHint
|
||||||
|
for _, hints := range providersHints {
|
||||||
|
// If hints is nil, insert a single, preferred any-numa hint into allProviderHints.
|
||||||
|
if len(hints) == 0 {
|
||||||
|
klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with any resource")
|
||||||
|
allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise, accumulate the hints for each resource type into allProviderHints.
|
||||||
|
for resource := range hints {
|
||||||
|
if hints[resource] == nil {
|
||||||
|
klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with resource '%s'", resource)
|
||||||
|
allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(hints[resource]) == 0 {
|
||||||
|
klog.Infof("[topologymanager] Hint Provider has no possible NUMA affinities for resource '%s'", resource)
|
||||||
|
allProviderHints = append(allProviderHints, []TopologyHint{{nil, false}})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
allProviderHints = append(allProviderHints, hints[resource])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Iterate over all permutations of hints in 'allProviderHints'. Merge the
|
||||||
|
// hints in each permutation by taking the bitwise-and of their affinity masks.
|
||||||
|
// Return the hint with the narrowest NUMANodeAffinity of all merged
|
||||||
|
// permutations that have at least one NUMA ID set. If no merged mask can be
|
||||||
|
// found that has at least one NUMA ID set, return the 'defaultAffinity'.
|
||||||
|
bestHint := TopologyHint{defaultAffinity, false}
|
||||||
|
iterateAllProviderTopologyHints(allProviderHints, func(permutation []TopologyHint) {
|
||||||
|
// Get the NUMANodeAffinity from each hint in the permutation and see if any
|
||||||
|
// of them encode unpreferred allocations.
|
||||||
|
preferred := true
|
||||||
|
var numaAffinities []bitmask.BitMask
|
||||||
|
for _, hint := range permutation {
|
||||||
|
if hint.NUMANodeAffinity == nil {
|
||||||
|
numaAffinities = append(numaAffinities, defaultAffinity)
|
||||||
|
} else {
|
||||||
|
numaAffinities = append(numaAffinities, hint.NUMANodeAffinity)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hint.Preferred {
|
||||||
|
preferred = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Special case PolicySingleNumaNode to only prefer hints where
|
||||||
|
// all providers have a single NUMA affinity set.
|
||||||
|
if policy != nil && policy.Name() == PolicySingleNumaNode && hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() > 1 {
|
||||||
|
preferred = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge the affinities using a bitwise-and operation.
|
||||||
|
mergedAffinity, _ := bitmask.NewBitMask(numaNodes...)
|
||||||
|
mergedAffinity.And(numaAffinities...)
|
||||||
|
|
||||||
|
// Build a mergedHintfrom the merged affinity mask, indicating if an
|
||||||
|
// preferred allocation was used to generate the affinity mask or not.
|
||||||
|
mergedHint := TopologyHint{mergedAffinity, preferred}
|
||||||
|
|
||||||
|
// Only consider mergedHints that result in a NUMANodeAffinity > 0 to
|
||||||
|
// replace the current bestHint.
|
||||||
|
if mergedHint.NUMANodeAffinity.Count() == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the current bestHint is non-preferred and the new mergedHint is
|
||||||
|
// preferred, always choose the preferred hint over the non-preferred one.
|
||||||
|
if mergedHint.Preferred && !bestHint.Preferred {
|
||||||
|
bestHint = mergedHint
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the current bestHint is preferred and the new mergedHint is
|
||||||
|
// non-preferred, never update bestHint, regardless of mergedHint's
|
||||||
|
// narowness.
|
||||||
|
if !mergedHint.Preferred && bestHint.Preferred {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// If mergedHint and bestHint has the same preference, only consider
|
||||||
|
// mergedHints that have a narrower NUMANodeAffinity than the
|
||||||
|
// NUMANodeAffinity in the current bestHint.
|
||||||
|
if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// In all other cases, update bestHint to the current mergedHint
|
||||||
|
bestHint = mergedHint
|
||||||
|
})
|
||||||
|
|
||||||
|
return bestHint
|
||||||
|
}
|
||||||
|
|||||||
@ -176,151 +176,6 @@ func (m *manager) accumulateProvidersHints(pod v1.Pod, container v1.Container) (
|
|||||||
return providersHints
|
return providersHints
|
||||||
}
|
}
|
||||||
|
|
||||||
// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'.
|
|
||||||
//
|
|
||||||
// This procedure is implemented as a recursive function over the set of hints
|
|
||||||
// in 'allproviderHints[i]'. It applies the function 'callback' to each
|
|
||||||
// permutation as it is found. It is the equivalent of:
|
|
||||||
//
|
|
||||||
// for i := 0; i < len(providerHints[0]); i++
|
|
||||||
// for j := 0; j < len(providerHints[1]); j++
|
|
||||||
// for k := 0; k < len(providerHints[2]); k++
|
|
||||||
// ...
|
|
||||||
// for z := 0; z < len(providerHints[-1]); z++
|
|
||||||
// permutation := []TopologyHint{
|
|
||||||
// providerHints[0][i],
|
|
||||||
// providerHints[1][j],
|
|
||||||
// providerHints[2][k],
|
|
||||||
// ...
|
|
||||||
// provideryHints[-1][z]
|
|
||||||
// }
|
|
||||||
// callback(permutation)
|
|
||||||
func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) {
|
|
||||||
// Internal helper function to accumulate the permutation before calling the callback.
|
|
||||||
var iterate func(i int, accum []TopologyHint)
|
|
||||||
iterate = func(i int, accum []TopologyHint) {
|
|
||||||
// Base case: we have looped through all providers and have a full permutation.
|
|
||||||
if i == len(allProviderHints) {
|
|
||||||
callback(accum)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Loop through all hints for provider 'i', and recurse to build the
|
|
||||||
// the permutation of this hint with all hints from providers 'i++'.
|
|
||||||
for j := range allProviderHints[i] {
|
|
||||||
iterate(i+1, append(accum, allProviderHints[i][j]))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
iterate(0, []TopologyHint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge the hints from all hint providers to find the best one.
|
|
||||||
func mergeProvidersHints(policy Policy, numaNodes []int, providersHints []map[string][]TopologyHint) TopologyHint {
|
|
||||||
// Set the default affinity as an any-numa affinity containing the list
|
|
||||||
// of NUMA Nodes available on this machine.
|
|
||||||
defaultAffinity, _ := bitmask.NewBitMask(numaNodes...)
|
|
||||||
|
|
||||||
// Loop through all hint providers and save an accumulated list of the
|
|
||||||
// hints returned by each hint provider. If no hints are provided, assume
|
|
||||||
// that provider has no preference for topology-aware allocation.
|
|
||||||
var allProviderHints [][]TopologyHint
|
|
||||||
for _, hints := range providersHints {
|
|
||||||
// If hints is nil, insert a single, preferred any-numa hint into allProviderHints.
|
|
||||||
if len(hints) == 0 {
|
|
||||||
klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with any resource")
|
|
||||||
allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Otherwise, accumulate the hints for each resource type into allProviderHints.
|
|
||||||
for resource := range hints {
|
|
||||||
if hints[resource] == nil {
|
|
||||||
klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with resource '%s'", resource)
|
|
||||||
allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(hints[resource]) == 0 {
|
|
||||||
klog.Infof("[topologymanager] Hint Provider has no possible NUMA affinities for resource '%s'", resource)
|
|
||||||
allProviderHints = append(allProviderHints, []TopologyHint{{nil, false}})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
allProviderHints = append(allProviderHints, hints[resource])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Iterate over all permutations of hints in 'allProviderHints'. Merge the
|
|
||||||
// hints in each permutation by taking the bitwise-and of their affinity masks.
|
|
||||||
// Return the hint with the narrowest NUMANodeAffinity of all merged
|
|
||||||
// permutations that have at least one NUMA ID set. If no merged mask can be
|
|
||||||
// found that has at least one NUMA ID set, return the 'defaultAffinity'.
|
|
||||||
bestHint := TopologyHint{defaultAffinity, false}
|
|
||||||
iterateAllProviderTopologyHints(allProviderHints, func(permutation []TopologyHint) {
|
|
||||||
// Get the NUMANodeAffinity from each hint in the permutation and see if any
|
|
||||||
// of them encode unpreferred allocations.
|
|
||||||
preferred := true
|
|
||||||
var numaAffinities []bitmask.BitMask
|
|
||||||
for _, hint := range permutation {
|
|
||||||
if hint.NUMANodeAffinity == nil {
|
|
||||||
numaAffinities = append(numaAffinities, defaultAffinity)
|
|
||||||
} else {
|
|
||||||
numaAffinities = append(numaAffinities, hint.NUMANodeAffinity)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !hint.Preferred {
|
|
||||||
preferred = false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Special case PolicySingleNumaNode to only prefer hints where
|
|
||||||
// all providers have a single NUMA affinity set.
|
|
||||||
if policy != nil && policy.Name() == PolicySingleNumaNode && hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() > 1 {
|
|
||||||
preferred = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge the affinities using a bitwise-and operation.
|
|
||||||
mergedAffinity, _ := bitmask.NewBitMask(numaNodes...)
|
|
||||||
mergedAffinity.And(numaAffinities...)
|
|
||||||
|
|
||||||
// Build a mergedHintfrom the merged affinity mask, indicating if an
|
|
||||||
// preferred allocation was used to generate the affinity mask or not.
|
|
||||||
mergedHint := TopologyHint{mergedAffinity, preferred}
|
|
||||||
|
|
||||||
// Only consider mergedHints that result in a NUMANodeAffinity > 0 to
|
|
||||||
// replace the current bestHint.
|
|
||||||
if mergedHint.NUMANodeAffinity.Count() == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the current bestHint is non-preferred and the new mergedHint is
|
|
||||||
// preferred, always choose the preferred hint over the non-preferred one.
|
|
||||||
if mergedHint.Preferred && !bestHint.Preferred {
|
|
||||||
bestHint = mergedHint
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the current bestHint is preferred and the new mergedHint is
|
|
||||||
// non-preferred, never update bestHint, regardless of mergedHint's
|
|
||||||
// narowness.
|
|
||||||
if !mergedHint.Preferred && bestHint.Preferred {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// If mergedHint and bestHint has the same preference, only consider
|
|
||||||
// mergedHints that have a narrower NUMANodeAffinity than the
|
|
||||||
// NUMANodeAffinity in the current bestHint.
|
|
||||||
if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// In all other cases, update bestHint to the current mergedHint
|
|
||||||
bestHint = mergedHint
|
|
||||||
})
|
|
||||||
|
|
||||||
return bestHint
|
|
||||||
}
|
|
||||||
|
|
||||||
// Collect Hints from hint providers and pass to policy to retrieve the best one.
|
// Collect Hints from hint providers and pass to policy to retrieve the best one.
|
||||||
func (m *manager) calculateAffinity(pod v1.Pod, container v1.Container) (TopologyHint, lifecycle.PodAdmitResult) {
|
func (m *manager) calculateAffinity(pod v1.Pod, container v1.Container) (TopologyHint, lifecycle.PodAdmitResult) {
|
||||||
providersHints := m.accumulateProvidersHints(pod, container)
|
providersHints := m.accumulateProvidersHints(pod, container)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user