diff --git a/pkg/kubelet/cm/topologymanager/policy_best_effort.go b/pkg/kubelet/cm/topologymanager/policy_best_effort.go index 97c61f031da..caf48cb30fe 100644 --- a/pkg/kubelet/cm/topologymanager/policy_best_effort.go +++ b/pkg/kubelet/cm/topologymanager/policy_best_effort.go @@ -17,6 +17,8 @@ limitations under the License. package topologymanager import ( + "k8s.io/klog" + "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask" "k8s.io/kubernetes/pkg/kubelet/lifecycle" ) @@ -50,3 +52,148 @@ func (p *bestEffortPolicy) Merge(providersHints []map[string][]TopologyHint) (To admit := p.canAdmitPodResult(&hint) return hint, admit } + +// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'. +// +// This procedure is implemented as a recursive function over the set of hints +// in 'allproviderHints[i]'. It applies the function 'callback' to each +// permutation as it is found. It is the equivalent of: +// +// for i := 0; i < len(providerHints[0]); i++ +// for j := 0; j < len(providerHints[1]); j++ +// for k := 0; k < len(providerHints[2]); k++ +// ... +// for z := 0; z < len(providerHints[-1]); z++ +// permutation := []TopologyHint{ +// providerHints[0][i], +// providerHints[1][j], +// providerHints[2][k], +// ... +// provideryHints[-1][z] +// } +// callback(permutation) +func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) { + // Internal helper function to accumulate the permutation before calling the callback. + var iterate func(i int, accum []TopologyHint) + iterate = func(i int, accum []TopologyHint) { + // Base case: we have looped through all providers and have a full permutation. + if i == len(allProviderHints) { + callback(accum) + return + } + + // Loop through all hints for provider 'i', and recurse to build the + // the permutation of this hint with all hints from providers 'i++'. + for j := range allProviderHints[i] { + iterate(i+1, append(accum, allProviderHints[i][j])) + } + } + iterate(0, []TopologyHint{}) +} + +// Merge the hints from all hint providers to find the best one. +func mergeProvidersHints(policy Policy, numaNodes []int, providersHints []map[string][]TopologyHint) TopologyHint { + // Set the default affinity as an any-numa affinity containing the list + // of NUMA Nodes available on this machine. + defaultAffinity, _ := bitmask.NewBitMask(numaNodes...) + + // Loop through all hint providers and save an accumulated list of the + // hints returned by each hint provider. If no hints are provided, assume + // that provider has no preference for topology-aware allocation. + var allProviderHints [][]TopologyHint + for _, hints := range providersHints { + // If hints is nil, insert a single, preferred any-numa hint into allProviderHints. + if len(hints) == 0 { + klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with any resource") + allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}}) + continue + } + + // Otherwise, accumulate the hints for each resource type into allProviderHints. + for resource := range hints { + if hints[resource] == nil { + klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with resource '%s'", resource) + allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}}) + continue + } + + if len(hints[resource]) == 0 { + klog.Infof("[topologymanager] Hint Provider has no possible NUMA affinities for resource '%s'", resource) + allProviderHints = append(allProviderHints, []TopologyHint{{nil, false}}) + continue + } + + allProviderHints = append(allProviderHints, hints[resource]) + } + } + + // Iterate over all permutations of hints in 'allProviderHints'. Merge the + // hints in each permutation by taking the bitwise-and of their affinity masks. + // Return the hint with the narrowest NUMANodeAffinity of all merged + // permutations that have at least one NUMA ID set. If no merged mask can be + // found that has at least one NUMA ID set, return the 'defaultAffinity'. + bestHint := TopologyHint{defaultAffinity, false} + iterateAllProviderTopologyHints(allProviderHints, func(permutation []TopologyHint) { + // Get the NUMANodeAffinity from each hint in the permutation and see if any + // of them encode unpreferred allocations. + preferred := true + var numaAffinities []bitmask.BitMask + for _, hint := range permutation { + if hint.NUMANodeAffinity == nil { + numaAffinities = append(numaAffinities, defaultAffinity) + } else { + numaAffinities = append(numaAffinities, hint.NUMANodeAffinity) + } + + if !hint.Preferred { + preferred = false + } + + // Special case PolicySingleNumaNode to only prefer hints where + // all providers have a single NUMA affinity set. + if policy != nil && policy.Name() == PolicySingleNumaNode && hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() > 1 { + preferred = false + } + } + + // Merge the affinities using a bitwise-and operation. + mergedAffinity, _ := bitmask.NewBitMask(numaNodes...) + mergedAffinity.And(numaAffinities...) + + // Build a mergedHintfrom the merged affinity mask, indicating if an + // preferred allocation was used to generate the affinity mask or not. + mergedHint := TopologyHint{mergedAffinity, preferred} + + // Only consider mergedHints that result in a NUMANodeAffinity > 0 to + // replace the current bestHint. + if mergedHint.NUMANodeAffinity.Count() == 0 { + return + } + + // If the current bestHint is non-preferred and the new mergedHint is + // preferred, always choose the preferred hint over the non-preferred one. + if mergedHint.Preferred && !bestHint.Preferred { + bestHint = mergedHint + return + } + + // If the current bestHint is preferred and the new mergedHint is + // non-preferred, never update bestHint, regardless of mergedHint's + // narowness. + if !mergedHint.Preferred && bestHint.Preferred { + return + } + + // If mergedHint and bestHint has the same preference, only consider + // mergedHints that have a narrower NUMANodeAffinity than the + // NUMANodeAffinity in the current bestHint. + if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) { + return + } + + // In all other cases, update bestHint to the current mergedHint + bestHint = mergedHint + }) + + return bestHint +} diff --git a/pkg/kubelet/cm/topologymanager/topology_manager.go b/pkg/kubelet/cm/topologymanager/topology_manager.go index 3600d3700db..4e452d71a00 100644 --- a/pkg/kubelet/cm/topologymanager/topology_manager.go +++ b/pkg/kubelet/cm/topologymanager/topology_manager.go @@ -176,151 +176,6 @@ func (m *manager) accumulateProvidersHints(pod v1.Pod, container v1.Container) ( return providersHints } -// Iterate over all permutations of hints in 'allProviderHints [][]TopologyHint'. -// -// This procedure is implemented as a recursive function over the set of hints -// in 'allproviderHints[i]'. It applies the function 'callback' to each -// permutation as it is found. It is the equivalent of: -// -// for i := 0; i < len(providerHints[0]); i++ -// for j := 0; j < len(providerHints[1]); j++ -// for k := 0; k < len(providerHints[2]); k++ -// ... -// for z := 0; z < len(providerHints[-1]); z++ -// permutation := []TopologyHint{ -// providerHints[0][i], -// providerHints[1][j], -// providerHints[2][k], -// ... -// provideryHints[-1][z] -// } -// callback(permutation) -func iterateAllProviderTopologyHints(allProviderHints [][]TopologyHint, callback func([]TopologyHint)) { - // Internal helper function to accumulate the permutation before calling the callback. - var iterate func(i int, accum []TopologyHint) - iterate = func(i int, accum []TopologyHint) { - // Base case: we have looped through all providers and have a full permutation. - if i == len(allProviderHints) { - callback(accum) - return - } - - // Loop through all hints for provider 'i', and recurse to build the - // the permutation of this hint with all hints from providers 'i++'. - for j := range allProviderHints[i] { - iterate(i+1, append(accum, allProviderHints[i][j])) - } - } - iterate(0, []TopologyHint{}) -} - -// Merge the hints from all hint providers to find the best one. -func mergeProvidersHints(policy Policy, numaNodes []int, providersHints []map[string][]TopologyHint) TopologyHint { - // Set the default affinity as an any-numa affinity containing the list - // of NUMA Nodes available on this machine. - defaultAffinity, _ := bitmask.NewBitMask(numaNodes...) - - // Loop through all hint providers and save an accumulated list of the - // hints returned by each hint provider. If no hints are provided, assume - // that provider has no preference for topology-aware allocation. - var allProviderHints [][]TopologyHint - for _, hints := range providersHints { - // If hints is nil, insert a single, preferred any-numa hint into allProviderHints. - if len(hints) == 0 { - klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with any resource") - allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}}) - continue - } - - // Otherwise, accumulate the hints for each resource type into allProviderHints. - for resource := range hints { - if hints[resource] == nil { - klog.Infof("[topologymanager] Hint Provider has no preference for NUMA affinity with resource '%s'", resource) - allProviderHints = append(allProviderHints, []TopologyHint{{nil, true}}) - continue - } - - if len(hints[resource]) == 0 { - klog.Infof("[topologymanager] Hint Provider has no possible NUMA affinities for resource '%s'", resource) - allProviderHints = append(allProviderHints, []TopologyHint{{nil, false}}) - continue - } - - allProviderHints = append(allProviderHints, hints[resource]) - } - } - - // Iterate over all permutations of hints in 'allProviderHints'. Merge the - // hints in each permutation by taking the bitwise-and of their affinity masks. - // Return the hint with the narrowest NUMANodeAffinity of all merged - // permutations that have at least one NUMA ID set. If no merged mask can be - // found that has at least one NUMA ID set, return the 'defaultAffinity'. - bestHint := TopologyHint{defaultAffinity, false} - iterateAllProviderTopologyHints(allProviderHints, func(permutation []TopologyHint) { - // Get the NUMANodeAffinity from each hint in the permutation and see if any - // of them encode unpreferred allocations. - preferred := true - var numaAffinities []bitmask.BitMask - for _, hint := range permutation { - if hint.NUMANodeAffinity == nil { - numaAffinities = append(numaAffinities, defaultAffinity) - } else { - numaAffinities = append(numaAffinities, hint.NUMANodeAffinity) - } - - if !hint.Preferred { - preferred = false - } - - // Special case PolicySingleNumaNode to only prefer hints where - // all providers have a single NUMA affinity set. - if policy != nil && policy.Name() == PolicySingleNumaNode && hint.NUMANodeAffinity != nil && hint.NUMANodeAffinity.Count() > 1 { - preferred = false - } - } - - // Merge the affinities using a bitwise-and operation. - mergedAffinity, _ := bitmask.NewBitMask(numaNodes...) - mergedAffinity.And(numaAffinities...) - - // Build a mergedHintfrom the merged affinity mask, indicating if an - // preferred allocation was used to generate the affinity mask or not. - mergedHint := TopologyHint{mergedAffinity, preferred} - - // Only consider mergedHints that result in a NUMANodeAffinity > 0 to - // replace the current bestHint. - if mergedHint.NUMANodeAffinity.Count() == 0 { - return - } - - // If the current bestHint is non-preferred and the new mergedHint is - // preferred, always choose the preferred hint over the non-preferred one. - if mergedHint.Preferred && !bestHint.Preferred { - bestHint = mergedHint - return - } - - // If the current bestHint is preferred and the new mergedHint is - // non-preferred, never update bestHint, regardless of mergedHint's - // narowness. - if !mergedHint.Preferred && bestHint.Preferred { - return - } - - // If mergedHint and bestHint has the same preference, only consider - // mergedHints that have a narrower NUMANodeAffinity than the - // NUMANodeAffinity in the current bestHint. - if !mergedHint.NUMANodeAffinity.IsNarrowerThan(bestHint.NUMANodeAffinity) { - return - } - - // In all other cases, update bestHint to the current mergedHint - bestHint = mergedHint - }) - - return bestHint -} - // Collect Hints from hint providers and pass to policy to retrieve the best one. func (m *manager) calculateAffinity(pod v1.Pod, container v1.Container) (TopologyHint, lifecycle.PodAdmitResult) { providersHints := m.accumulateProvidersHints(pod, container)