From b28c1392d7edfa07029dc76b6f9bfeef4dfc8856 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Tue, 23 Nov 2021 20:27:54 +0000 Subject: [PATCH 01/13] Round the CPUManager mean and stddev calculations to the nearest 1000th Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index df6cc3b8265..696bafca8dd 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -65,7 +65,8 @@ func mean(xs []int) float64 { for _, x := range xs { sum += float64(x) } - return sum / float64(len(xs)) + m := sum / float64(len(xs)) + return math.Round(m*1000) / 1000 } func standardDeviation(xs []int) float64 { @@ -74,7 +75,8 @@ func standardDeviation(xs []int) float64 { for _, x := range xs { sum += (float64(x) - m) * (float64(x) - m) } - return math.Sqrt(sum / float64(len(xs))) + s := math.Sqrt(sum / float64(len(xs))) + return math.Round(s*1000) / 1000 } func min(x, y int) int { From c8559bc43e507d7f7759c7b53ea12ae70c017fa0 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 24 Nov 2021 14:10:50 +0000 Subject: [PATCH 02/13] Short-circuit CPUManager distribute NUMA algo for unusable cpuGroupSize Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 696bafca8dd..0d24ad1c1a8 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -534,6 +534,14 @@ func takeByTopologyNUMAPacked(topo *topology.CPUTopology, availableCPUs cpuset.C // important, for example, to ensure that all CPUs (i.e. all hyperthreads) from // a single core are allocated together. func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int, cpuGroupSize int) (cpuset.CPUSet, error) { + // If the number of CPUs requested cannot be handed out in chunks of + // 'cpuGroupSize', then we just call out the packing algorithm since we + // can't distribute CPUs in this chunk size. + if (numCPUs % cpuGroupSize) != 0 { + return takeByTopologyNUMAPacked(topo, availableCPUs, numCPUs) + } + + // Otherwise build an accumulator to start allocating CPUs from. acc := newCPUAccumulator(topo, availableCPUs, numCPUs) if acc.isSatisfied() { return acc.result, nil From 446c58e0e7b887b85d24ba50648d1b705810b005 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 24 Nov 2021 00:17:31 +0000 Subject: [PATCH 03/13] Ensure we balance across *all* NUMA nodes in NUMA distribution algo Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 0d24ad1c1a8..2f8299c211f 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -605,13 +605,16 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu } // Calculate how many CPUs will be available on each NUMA node in - // 'combo' after allocating an even distribution of CPU groups of - // size 'cpuGroupSize' from them. This will be used in the "balance - // score" calculation to help decide if this combo should - // ultimately be chosen. - availableAfterAllocation := make(mapIntInt, len(combo)) + // the system after allocating an even distribution of CPU groups + // of size 'cpuGroupSize' from each NUMA node in 'combo'. This will + // be used in the "balance score" calculation to help decide if + // this combo should ultimately be chosen. + availableAfterAllocation := make(mapIntInt, len(numas)) + for _, numa := range numas { + availableAfterAllocation[numa] = acc.details.CPUsInNUMANodes(numa).Size() + } for _, numa := range combo { - availableAfterAllocation[numa] = acc.details.CPUsInNUMANodes(numa).Size() - distribution + availableAfterAllocation[numa] -= distribution } // Check if there are any remaining CPUs to distribute across the @@ -648,7 +651,7 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu } // Calculate the "balance score" as the standard deviation of - // the number of CPUs available on all NUMA nodes in 'combo' + // the number of CPUs available on all NUMA nodes in the system // after the remainder CPUs have been allocated across 'subset' // in groups of size 'cpuGroupSize'. balance := standardDeviation(availableAfterAllocation.Values()) From 4008ea0b4c1835152b587bd2cef7e313e6d2c452 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Tue, 23 Nov 2021 21:59:46 +0000 Subject: [PATCH 04/13] Fix bug in CPUManager map.Keys() and map.Values() implementations Previously these would return lists that were too long because we appended to pre-initialized lists with a specific size. Since the primary place these functions are used is in the mean and standard deviation calculations for the NUMA distribution algorithm, it meant that the results of these calculations were often incorrect. As a result, some of the unit tests we have are actually incorrect (because the results we expect do not actually produce the best balanced distribution of CPUs across all NUMA nodes for the input provided). These tests will be patched up in subsequent commits. Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 2f8299c211f..eade5f08ecd 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -45,7 +45,7 @@ func (m mapIntInt) Clone() mapIntInt { } func (m mapIntInt) Keys() []int { - keys := make([]int, len(m)) + var keys []int for k := range m { keys = append(keys, k) } @@ -53,7 +53,7 @@ func (m mapIntInt) Keys() []int { } func (m mapIntInt) Values() []int { - values := make([]int, len(m)) + var values []int for _, v := range m { values = append(values, v) } From 67f719cb1dd8cb66135b93bfb09c3b09859d38d7 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Tue, 23 Nov 2021 21:59:46 +0000 Subject: [PATCH 05/13] Fix unit tests following bug fix in CPUManager for map functions (1/2) This fixes two related tests to better test our "balanced" distribution algorithm. The first test originally provided an input with the following number of CPUs available on each NUMA node: Node 0: 16 Node 1: 20 Node 2: 20 Node 3: 20 It then attempted to distribute 48 CPUs across them with an expectation that each of the first 3 NUMA nodes would have 16 CPUs taken from them (leaving Node 0 with no more CPUs in the end). This would have resulted in the following amount of CPUs on each node: Node 0: 0 Node 1: 4 Node 2: 4 Node 3: 20 Which results in a standard deviation of 7.6811 However, a more balanced solution would actually be to pull 16 CPUs from NUMA nodes 1, 2, and 3, and leave 0 untouched, i.e.: Node 0: 16 Node 1: 4 Node 2: 4 Node 3: 4 Which results in a standard deviation of 5.1961524227066 To fix this test we changed the original number of available CPUs to start with 4 less CPUs on NUMA node 3, and 2 more CPUs on NUMA node 0, i.e.: Node 0: 18 Node 1: 20 Node 2: 20 Node 3: 16 So that we end up with a result of: Node 0: 2 Node 1: 4 Node 2: 4 Node 3: 16 Which pulls the CPUs from where we want and results in a standard deviation of 5.5452 For the second test, we simply reverse the number of CPUs available for Nodes 0 and 3 as: Node 0: 16 Node 1: 20 Node 2: 20 Node 3: 18 Which forces the allocation to happen just as it did for the first test, except now on NUMA nodes 1, 2, and 3 instead of NUMA nodes 0,1, and 2. Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go index 59059feb0cc..e57351d4890 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go @@ -721,22 +721,22 @@ func commonTakeByTopologyExtendedTestCases(t *testing.T) []takeByTopologyExtende mustParseCPUSet(t, "0-7,10-17,20-27,40-47,50-57,60-67"), }, { - "allocate 24 full cores with 8 distributed across the first 3 NUMA nodes (filling the first NUMA node)", + "allocate 24 full cores with 8 distributed across the first 3 NUMA nodes (taking all but 2 from the first NUMA node)", topoDualSocketMultiNumaPerSocketHT, - mustParseCPUSet(t, "2-39,42-79"), + mustParseCPUSet(t, "1-29,32-39,41-69,72-79"), 48, 1, "", - mustParseCPUSet(t, "2-9,10-17,20-27,42-49,50-57,60-67"), + mustParseCPUSet(t, "1-8,10-17,20-27,41-48,50-57,60-67"), }, { - "allocate 24 full cores with 8 distributed across the last 3 NUMA nodes (no room on the first NUMA node to distribute)", + "allocate 24 full cores with 8 distributed across the last 3 NUMA nodes (even though all 8 could be allocated from the first NUMA node)", topoDualSocketMultiNumaPerSocketHT, - mustParseCPUSet(t, "3-39,43-79"), + mustParseCPUSet(t, "2-29,31-39,42-69,71-79"), 48, 1, "", - mustParseCPUSet(t, "10-17,20-27,30-37,50-57,60-67,70-77"), + mustParseCPUSet(t, "10-17,20-27,31-38,50-57,60-67,71-78"), }, { "allocate 8 full cores with 2 distributed across each NUMA node", From 209cd20548a469cb3ddb8d2cc1a50f19677a7561 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 24 Nov 2021 00:20:21 +0000 Subject: [PATCH 06/13] Fix unit tests following bug fix in CPUManager for map functions (2/2) Now that the algorithm for balancing CPU distributions across NUMA nodes is correct, this test actually behaves differently for the "packed" vs. "distributed" allocation algorithms (as it should). In the "packed" case we need to ensure that CPUs are allocated such that they are packed onto cores. Since one CPU is already allocated from a core on NUMA node 0, we want the next CPU to be its hyperthreaded pair (even though the first available CPU id is on Socket 1). In the "distributed" case, however, we want to ensure CPUs are allocated such that we have an balanced distribution of CPUs across all NUMA nodes. This points to allocating from Socket 1 if the only other CPU allocated has been done on Socket 0. To allow CPUs allocations to be packed onto full cores, one can allocate them from the "distributed" algorithm with a 'cpuGroupSize' equal to the number of hypthreads per core (in this case 2). We added an explicit test case for this, demonstrating that we get the same result as the "packed" algorithm does, even though the "distributed" algorithm is in use. Signed-off-by: Kevin Klues --- .../cm/cpumanager/cpu_assignment_test.go | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go index e57351d4890..b1cb9f7878a 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go @@ -573,14 +573,6 @@ func commonTakeByTopologyTestCases(t *testing.T) []takeByTopologyTestCase { "", cpuset.NewCPUSet(2, 6), }, - { - "take one cpu from dual socket with HT - core from Socket 0", - topoDualSocketHT, - cpuset.NewCPUSet(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), - 1, - "", - cpuset.NewCPUSet(2), - }, { "take a socket of cpus from dual socket with HT", topoDualSocketHT, @@ -635,6 +627,14 @@ func commonTakeByTopologyTestCases(t *testing.T) []takeByTopologyTestCase { func TestTakeByTopologyNUMAPacked(t *testing.T) { testCases := commonTakeByTopologyTestCases(t) testCases = append(testCases, []takeByTopologyTestCase{ + { + "take one cpu from dual socket with HT - core from Socket 0", + topoDualSocketHT, + cpuset.NewCPUSet(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), + 1, + "", + cpuset.NewCPUSet(2), + }, { "allocate 4 full cores with 3 coming from the first NUMA node (filling it up) and 1 coming from the second NUMA node", topoDualSocketHT, @@ -764,6 +764,24 @@ func commonTakeByTopologyExtendedTestCases(t *testing.T) []takeByTopologyExtende func TestTakeByTopologyNUMADistributed(t *testing.T) { testCases := commonTakeByTopologyExtendedTestCases(t) testCases = append(testCases, []takeByTopologyExtendedTestCase{ + { + "take one cpu from dual socket with HT - core from Socket 0", + topoDualSocketHT, + cpuset.NewCPUSet(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), + 1, + 1, + "", + cpuset.NewCPUSet(1), + }, + { + "take one cpu from dual socket with HT - core from Socket 0 - cpuGroupSize 2", + topoDualSocketHT, + cpuset.NewCPUSet(1, 2, 3, 4, 5, 7, 8, 9, 10, 11), + 1, + 2, + "", + cpuset.NewCPUSet(2), + }, { "allocate 13 full cores distributed across the first 2 NUMA nodes", topoDualSocketMultiNumaPerSocketHT, From a160d9a8cd4b4c2154ba72ca25d7e37d3de879b4 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 24 Nov 2021 00:56:25 +0000 Subject: [PATCH 07/13] Fix CPUManager algo to calculate min NUMA nodes needed for distribution Previously the algorithm was too restrictive because it tried to calculate the minimum based on the number of *available* NUMA nodes and the number of *available* CPUs on those NUMA nodes. Since there was no (easy) way to tell how many CPUs an individual NUMA node happened to have, the average across them was used. Using this value however, could result in thinking you need more NUMA nodes to possibly satisfy a request than you actually do. By using the *total* number of NUMA nodes and CPUs per NUMA node, we can get the true minimum number of nodes required to satisfy a request. For a given "current" allocation this may not be the true minimum, but its better to start with fewer and move up than to start with too many and miss out on a better option. Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index eade5f08ecd..fe6e1b4187d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -367,20 +367,25 @@ func (a *cpuAccumulator) takeRemainingCPUs() { } func (a *cpuAccumulator) rangeNUMANodesNeededToSatisfy(cpuGroupSize int) (int, int) { + // Get the total number of NUMA nodes in the system. + numNUMANodes := a.topo.CPUDetails.NUMANodes().Size() + // Get the total number of NUMA nodes that have CPUs available on them. numNUMANodesAvailable := a.details.NUMANodes().Size() - // Get the total number of CPUs available across all NUMA nodes. - numCPUsAvailable := a.details.CPUs().Size() + // Get the total number of CPUs in the system. + numCPUs := a.topo.CPUDetails.CPUs().Size() + + // Get the total number of 'cpuGroups' in the system. + numCPUGroups := (numCPUs-1)/cpuGroupSize + 1 + + // Calculate the number of 'cpuGroups' per NUMA Node in the system (rounding up). + numCPUGroupsPerNUMANode := (numCPUGroups-1)/numNUMANodes + 1 // Calculate the number of available 'cpuGroups' across all NUMA nodes as // well as the number of 'cpuGroups' that need to be allocated (rounding up). - numCPUGroupsAvailable := (numCPUsAvailable-1)/cpuGroupSize + 1 numCPUGroupsNeeded := (a.numCPUsNeeded-1)/cpuGroupSize + 1 - // Calculate the number of available 'cpuGroups' per NUMA Node (rounding up). - numCPUGroupsPerNUMANode := (numCPUGroupsAvailable-1)/numNUMANodesAvailable + 1 - // Calculate the minimum number of numa nodes required to satisfy the // allocation (rounding up). minNUMAs := (numCPUGroupsNeeded-1)/numCPUGroupsPerNUMANode + 1 From cfacc22459f319dc0e3a1b6febae0b592e9d9fcc Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Tue, 23 Nov 2021 20:31:04 +0000 Subject: [PATCH 08/13] Allow the map.Values() function in the CPUManager to take a set of keys Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index fe6e1b4187d..810380fa0a8 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -52,10 +52,13 @@ func (m mapIntInt) Keys() []int { return keys } -func (m mapIntInt) Values() []int { +func (m mapIntInt) Values(keys ...int) []int { + if keys == nil { + keys = m.Keys() + } var values []int - for _, v := range m { - values = append(values, v) + for _, k := range keys { + values = append(values, m[k]) } return values } From dc4430b6633e59ef53849dc30bf9241caba06af6 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Tue, 23 Nov 2021 20:28:46 +0000 Subject: [PATCH 09/13] Add a sum() helper to the CPUManager cpuassignment logic Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 810380fa0a8..578b0415549 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -63,6 +63,14 @@ func (m mapIntInt) Values(keys ...int) []int { return values } +func sum(xs []int) int { + var s int + for _, x := range xs { + s += x + } + return s +} + func mean(xs []int) float64 { var sum float64 for _, x := range xs { From 5317a2e2acbb0c72c99a0cea069c47d3f91bf27b Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Mon, 22 Nov 2021 18:17:39 +0000 Subject: [PATCH 10/13] Fix error handling in CPUManager distribute NUMA tests Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go index b1cb9f7878a..98c6ae9e41d 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go @@ -841,8 +841,14 @@ func TestTakeByTopologyNUMADistributed(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { result, err := takeByTopologyNUMADistributed(tc.topo, tc.availableCPUs, tc.numCPUs, tc.cpuGroupSize) - if tc.expErr != "" && err.Error() != tc.expErr { - t.Errorf("expected error to be [%v] but it was [%v]", tc.expErr, err) + if err != nil { + if tc.expErr == "" { + t.Errorf("unexpected error [%v]", err) + } + if tc.expErr != "" && err.Error() != tc.expErr { + t.Errorf("expected error to be [%v] but it was [%v]", tc.expErr, err) + } + return } if !result.Equals(tc.expResult) { t.Errorf("expected result [%s] to equal [%s]", result, tc.expResult) From 031f11513dd5007d1f631b59c5d6eb63a73685bf Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Mon, 22 Nov 2021 18:18:26 +0000 Subject: [PATCH 11/13] Fix accounting bug in CPUManager distribute NUMA policy Without this fix, the algorithm may decide to allocate "remainder" CPUs from a NUMA node that has no more CPUs to allocate. Moreover, it was only considering allocation of remainder CPUs from NUMA nodes such that each NUMA node in the remainderSet could only allocate 1 (i.e. 'cpuGroupSize') more CPUs. With these two issues in play, one could end up with an accounting error where not enough CPUs were allocated by the time the algorithm runs to completion. The updated algorithm will now omit any NUMA nodes that have 0 CPUs left from the set of NUMA nodes considered for allocating remainder CPUs. Additionally, we now consider *all* combinations of nodes from the remainder set of size 1..len(remainderSet). This allows us to find a better solution if allocating CPUs from a smaller set leads to a more balanced allocation. Finally, we loop through all NUMA nodes 1-by-1 in the remainderSet until all rmeainer CPUs have been accounted for and allocated. This ensure that we will not hit an accounting error later on because we explicitly remove CPUs from the remainder set until there are none left. A follow-on commit adds a set of unit tests that will fail before these changes, but succeeds after them. Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment.go | 96 +++++++++++++++------ 1 file changed, 70 insertions(+), 26 deletions(-) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment.go b/pkg/kubelet/cm/cpumanager/cpu_assignment.go index 578b0415549..01ddf57bb7b 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment.go @@ -638,9 +638,20 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu // size 'cpuGroupSize'. remainder := numCPUs - (distribution * len(combo)) + // Get a list of NUMA nodes to consider pulling the remainder CPUs + // from. This list excludes NUMA nodes that don't have at least + // 'cpuGroupSize' CPUs available after being allocated + // 'distribution' number of CPUs. + var remainderCombo []int + for _, numa := range combo { + if availableAfterAllocation[numa] >= cpuGroupSize { + remainderCombo = append(remainderCombo, numa) + } + } + // Declare a set of local variables to help track the "balance - // scores" calculated when using different subsets of 'combo' to - // allocate remainder CPUs from. + // scores" calculated when using different subsets of + // 'remainderCombo' to allocate remainder CPUs from. var bestLocalBalance float64 = math.MaxFloat64 var bestLocalRemainder []int = nil @@ -653,31 +664,54 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu } // Otherwise, find the best "balance score" when allocating the - // remainder CPUs across different subsets of NUMA nodes in 'combo'. + // remainder CPUs across different subsets of NUMA nodes in 'remainderCombo'. // These remainder CPUs are handed out in groups of size 'cpuGroupSize'. - acc.iterateCombinations(combo, remainder/cpuGroupSize, func(subset []int) LoopControl { - // Make a local copy of 'availableAfterAllocation'. - availableAfterAllocation := availableAfterAllocation.Clone() + // We start from k=len(remainderCombo) and walk down to k=1 so that + // we continue to distribute CPUs as much as possible across + // multiple NUMA nodes. + for k := len(remainderCombo); remainder > 0 && k >= 1; k-- { + acc.iterateCombinations(remainderCombo, k, func(subset []int) LoopControl { + // Make a local copy of 'remainder'. + remainder := remainder - // For all NUMA nodes in 'subset', remove another - // 'cpuGroupSize' number of CPUs (to account for any remainder - // CPUs that will be allocated on them). - for _, numa := range subset { - availableAfterAllocation[numa] -= cpuGroupSize - } + // Make a local copy of 'availableAfterAllocation'. + availableAfterAllocation := availableAfterAllocation.Clone() - // Calculate the "balance score" as the standard deviation of - // the number of CPUs available on all NUMA nodes in the system - // after the remainder CPUs have been allocated across 'subset' - // in groups of size 'cpuGroupSize'. - balance := standardDeviation(availableAfterAllocation.Values()) - if balance < bestLocalBalance { - bestLocalBalance = balance - bestLocalRemainder = subset - } + // If this subset is not capable of allocating all + // remainder CPUs, continue to the next one. + if sum(availableAfterAllocation.Values(subset...)) < remainder { + return Continue + } - return Continue - }) + // For all NUMA nodes in 'subset', walk through them, + // removing 'cpuGroupSize' number of CPUs from each + // until all remainder CPUs have been accounted for. + for remainder > 0 { + for _, numa := range subset { + if remainder == 0 { + break + } + if availableAfterAllocation[numa] < cpuGroupSize { + continue + } + availableAfterAllocation[numa] -= cpuGroupSize + remainder -= cpuGroupSize + } + } + + // Calculate the "balance score" as the standard deviation + // of the number of CPUs available on all NUMA nodes in the + // system after the remainder CPUs have been allocated + // across 'subset' in groups of size 'cpuGroupSize'. + balance := standardDeviation(availableAfterAllocation.Values()) + if balance < bestLocalBalance { + bestLocalBalance = balance + bestLocalRemainder = subset + } + + return Continue + }) + } // If the best "balance score" for this combo is less than the // lowest "balance score" of all previous combos, then update this @@ -709,9 +743,19 @@ func takeByTopologyNUMADistributed(topo *topology.CPUTopology, availableCPUs cpu // Then allocate any remaining CPUs in groups of size 'cpuGroupSize' // from each NUMA node in the remainder set. - for _, numa := range bestRemainder { - cpus, _ := takeByTopologyNUMAPacked(acc.topo, acc.details.CPUsInNUMANodes(numa), cpuGroupSize) - acc.take(cpus) + remainder := numCPUs - (distribution * len(bestCombo)) + for remainder > 0 { + for _, numa := range bestRemainder { + if remainder == 0 { + break + } + if acc.details.CPUsInNUMANodes(numa).Size() < cpuGroupSize { + continue + } + cpus, _ := takeByTopologyNUMAPacked(acc.topo, acc.details.CPUsInNUMANodes(numa), cpuGroupSize) + acc.take(cpus) + remainder -= cpuGroupSize + } } // If we haven't allocated all of our CPUs at this point, then something From e284c74d93d585ee31d764608f6194aab2a18b72 Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 24 Nov 2021 19:16:25 +0000 Subject: [PATCH 12/13] Add unit test for CPUManager distribute NUMA algorithm verifying fixes Before Change: "test" description="ensure bestRemainder chosen with NUMA nodes that have enough CPUs to satisfy the request" "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[0 1] distribution=8 remainder=2 available=[-1 -1 0 6] balance=2.915 "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[0 2] distribution=8 remainder=2 available=[-1 0 -1 6] balance=2.915 "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[0 3] distribution=8 remainder=2 available=[5 -1 0 0] balance=2.345 "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[1 2] distribution=8 remainder=2 available=[0 -1 -1 6] balance=2.915 "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[1 3] distribution=8 remainder=2 available=[0 -1 0 5] balance=2.345 "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[2 3] distribution=8 remainder=2 available=[0 0 -1 5] balance=2.345 "bestCombo found" distribution=8 bestCombo=[0 1 2 3] bestRemainder=[0 3] --- FAIL: TestTakeByTopologyNUMADistributed (0.01s) --- FAIL: TestTakeByTopologyNUMADistributed/ensure_bestRemainder_chosen_with_NUMA_nodes_that_have_enough_CPUs_to_satisfy_the_request (0.00s) cpu_assignment_test.go:867: unexpected error [accounting error, not enough CPUs allocated, remaining: 1] After Change: "test" description="ensure bestRemainder chosen with NUMA nodes that have enough CPUs to satisfy the request" "combo remainderSet balance" combo=[0 1 2 3] remainderSet=[3] distribution=8 remainder=2 available=[0 0 0 4] balance=1.732 "bestCombo found" distribution=8 bestCombo=[0 1 2 3] bestRemainder=[3] SUCCESS Signed-off-by: Kevin Klues --- pkg/kubelet/cm/cpumanager/cpu_assignment_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go index 98c6ae9e41d..ade2dddaac3 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go @@ -836,6 +836,15 @@ func TestTakeByTopologyNUMADistributed(t *testing.T) { "", mustParseCPUSet(t, "0-7,10-16,20-27,30-37,40-47,50-56,60-67,70-77"), }, + { + "ensure bestRemainder chosen with NUMA nodes that have enough CPUs to satisfy the request", + topoDualSocketMultiNumaPerSocketHT, + mustParseCPUSet(t, "0-3,10-13,20-23,30-36,40-43,50-53,60-63,70-76"), + 34, + 1, + "", + mustParseCPUSet(t, "0-3,10-13,20-23,30-34,40-43,50-53,60-63,70-74"), + }, }...) for _, tc := range testCases { From f8511877e262015b89702357fce2f2da90e7fe2c Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Wed, 24 Nov 2021 20:45:53 +0000 Subject: [PATCH 13/13] Add regression test for CPUManager distribute NUMA algorithm We witnessed this exact allocation attempt in a live cluster and witnessed the algorithm fail with an accounting error. This test was added to verify that this case is now handled by the updates to the algorithm and that we don't regress from it in the future. "test" description="ensure previous failure encountered on live machine has been fixed (1/1)" "combo remainderSet balance" combo=[2 4 6] remainderSet=[2 4 6] distribution=9 remainder=1 available=[14 2 4 4 0 3 4 1] balance=4.031 "combo remainderSet balance" combo=[2 4 6] remainderSet=[2 4] distribution=9 remainder=1 available=[0 3 4 1 14 2 4 4] balance=4.031 "combo remainderSet balance" combo=[2 4 6] remainderSet=[2 6] distribution=9 remainder=1 available=[1 14 2 4 4 0 3 4] balance=4.031 "combo remainderSet balance" combo=[2 4 6] remainderSet=[4 6] distribution=9 remainder=1 available=[1 3 4 0 14 2 4 4] balance=4.031 "combo remainderSet balance" combo=[2 4 6] remainderSet=[2] distribution=9 remainder=1 available=[4 0 3 4 1 14 2 4] balance=4.031 "combo remainderSet balance" combo=[2 4 6] remainderSet=[4] distribution=9 remainder=1 available=[3 4 0 14 2 4 4 1] balance=4.031 "combo remainderSet balance" combo=[2 4 6] remainderSet=[6] distribution=9 remainder=1 available=[1 13 2 4 4 1 3 4] balance=3.606 "bestCombo found" distribution=9 bestCombo=[2 4 6] bestRemainder=[6] Signed-off-by: Kevin Klues --- .../cm/cpumanager/cpu_assignment_test.go | 9 + pkg/kubelet/cm/cpumanager/policy_test.go | 282 ++++++++++++++++++ 2 files changed, 291 insertions(+) diff --git a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go index ade2dddaac3..1f9dd09b66f 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_assignment_test.go @@ -845,6 +845,15 @@ func TestTakeByTopologyNUMADistributed(t *testing.T) { "", mustParseCPUSet(t, "0-3,10-13,20-23,30-34,40-43,50-53,60-63,70-74"), }, + { + "ensure previous failure encountered on live machine has been fixed (1/1)", + topoDualSocketMultiNumaPerSocketHTLarge, + mustParseCPUSet(t, "0,128,30,31,158,159,43-47,171-175,62,63,190,191,75-79,203-207,94,96,222,223,101-111,229-239,126,127,254,255"), + 28, + 1, + "", + mustParseCPUSet(t, "43-47,75-79,96,101-105,171-174,203-206,229-232"), + }, }...) for _, tc := range testCases { diff --git a/pkg/kubelet/cm/cpumanager/policy_test.go b/pkg/kubelet/cm/cpumanager/policy_test.go index 2cd681c395f..02f0898063a 100644 --- a/pkg/kubelet/cm/cpumanager/policy_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_test.go @@ -610,4 +610,286 @@ var ( 79: {CoreID: 39, SocketID: 3, NUMANodeID: 1}, }, } + + /* + Topology from dual AMD EPYC 7742 64-Core Processor; lscpu excerpt + CPU(s): 256 + On-line CPU(s) list: 0-255 + Thread(s) per core: 2 + Core(s) per socket: 64 + Socket(s): 2 + NUMA node(s): 8 (NPS=4) + NUMA node0 CPU(s): 0-15,128-143 + NUMA node1 CPU(s): 16-31,144-159 + NUMA node2 CPU(s): 32-47,160-175 + NUMA node3 CPU(s): 48-63,176-191 + NUMA node4 CPU(s): 64-79,192-207 + NUMA node5 CPU(s): 80-95,208-223 + NUMA node6 CPU(s): 96-111,224-239 + NUMA node7 CPU(s): 112-127,240-255 + */ + topoDualSocketMultiNumaPerSocketHTLarge = &topology.CPUTopology{ + NumCPUs: 256, + NumSockets: 2, + NumCores: 128, + NumNUMANodes: 8, + CPUDetails: map[int]topology.CPUInfo{ + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 4, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 5, SocketID: 0, NUMANodeID: 0}, + 6: {CoreID: 6, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 7, SocketID: 0, NUMANodeID: 0}, + 8: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 10: {CoreID: 10, SocketID: 0, NUMANodeID: 0}, + 11: {CoreID: 11, SocketID: 0, NUMANodeID: 0}, + 12: {CoreID: 12, SocketID: 0, NUMANodeID: 0}, + 13: {CoreID: 13, SocketID: 0, NUMANodeID: 0}, + 14: {CoreID: 14, SocketID: 0, NUMANodeID: 0}, + 15: {CoreID: 15, SocketID: 0, NUMANodeID: 0}, + 16: {CoreID: 16, SocketID: 0, NUMANodeID: 1}, + 17: {CoreID: 17, SocketID: 0, NUMANodeID: 1}, + 18: {CoreID: 18, SocketID: 0, NUMANodeID: 1}, + 19: {CoreID: 19, SocketID: 0, NUMANodeID: 1}, + 20: {CoreID: 20, SocketID: 0, NUMANodeID: 1}, + 21: {CoreID: 21, SocketID: 0, NUMANodeID: 1}, + 22: {CoreID: 22, SocketID: 0, NUMANodeID: 1}, + 23: {CoreID: 23, SocketID: 0, NUMANodeID: 1}, + 24: {CoreID: 24, SocketID: 0, NUMANodeID: 1}, + 25: {CoreID: 25, SocketID: 0, NUMANodeID: 1}, + 26: {CoreID: 26, SocketID: 0, NUMANodeID: 1}, + 27: {CoreID: 27, SocketID: 0, NUMANodeID: 1}, + 28: {CoreID: 28, SocketID: 0, NUMANodeID: 1}, + 29: {CoreID: 29, SocketID: 0, NUMANodeID: 1}, + 30: {CoreID: 30, SocketID: 0, NUMANodeID: 1}, + 31: {CoreID: 31, SocketID: 0, NUMANodeID: 1}, + 32: {CoreID: 32, SocketID: 0, NUMANodeID: 2}, + 33: {CoreID: 33, SocketID: 0, NUMANodeID: 2}, + 34: {CoreID: 34, SocketID: 0, NUMANodeID: 2}, + 35: {CoreID: 35, SocketID: 0, NUMANodeID: 2}, + 36: {CoreID: 36, SocketID: 0, NUMANodeID: 2}, + 37: {CoreID: 37, SocketID: 0, NUMANodeID: 2}, + 38: {CoreID: 38, SocketID: 0, NUMANodeID: 2}, + 39: {CoreID: 39, SocketID: 0, NUMANodeID: 2}, + 40: {CoreID: 40, SocketID: 0, NUMANodeID: 2}, + 41: {CoreID: 41, SocketID: 0, NUMANodeID: 2}, + 42: {CoreID: 42, SocketID: 0, NUMANodeID: 2}, + 43: {CoreID: 43, SocketID: 0, NUMANodeID: 2}, + 44: {CoreID: 44, SocketID: 0, NUMANodeID: 2}, + 45: {CoreID: 45, SocketID: 0, NUMANodeID: 2}, + 46: {CoreID: 46, SocketID: 0, NUMANodeID: 2}, + 47: {CoreID: 47, SocketID: 0, NUMANodeID: 2}, + 48: {CoreID: 48, SocketID: 0, NUMANodeID: 3}, + 49: {CoreID: 49, SocketID: 0, NUMANodeID: 3}, + 50: {CoreID: 50, SocketID: 0, NUMANodeID: 3}, + 51: {CoreID: 51, SocketID: 0, NUMANodeID: 3}, + 52: {CoreID: 52, SocketID: 0, NUMANodeID: 3}, + 53: {CoreID: 53, SocketID: 0, NUMANodeID: 3}, + 54: {CoreID: 54, SocketID: 0, NUMANodeID: 3}, + 55: {CoreID: 55, SocketID: 0, NUMANodeID: 3}, + 56: {CoreID: 56, SocketID: 0, NUMANodeID: 3}, + 57: {CoreID: 57, SocketID: 0, NUMANodeID: 3}, + 58: {CoreID: 58, SocketID: 0, NUMANodeID: 3}, + 59: {CoreID: 59, SocketID: 0, NUMANodeID: 3}, + 60: {CoreID: 60, SocketID: 0, NUMANodeID: 3}, + 61: {CoreID: 61, SocketID: 0, NUMANodeID: 3}, + 62: {CoreID: 62, SocketID: 0, NUMANodeID: 3}, + 63: {CoreID: 63, SocketID: 0, NUMANodeID: 3}, + 64: {CoreID: 64, SocketID: 1, NUMANodeID: 4}, + 65: {CoreID: 65, SocketID: 1, NUMANodeID: 4}, + 66: {CoreID: 66, SocketID: 1, NUMANodeID: 4}, + 67: {CoreID: 67, SocketID: 1, NUMANodeID: 4}, + 68: {CoreID: 68, SocketID: 1, NUMANodeID: 4}, + 69: {CoreID: 69, SocketID: 1, NUMANodeID: 4}, + 70: {CoreID: 70, SocketID: 1, NUMANodeID: 4}, + 71: {CoreID: 71, SocketID: 1, NUMANodeID: 4}, + 72: {CoreID: 72, SocketID: 1, NUMANodeID: 4}, + 73: {CoreID: 73, SocketID: 1, NUMANodeID: 4}, + 74: {CoreID: 74, SocketID: 1, NUMANodeID: 4}, + 75: {CoreID: 75, SocketID: 1, NUMANodeID: 4}, + 76: {CoreID: 76, SocketID: 1, NUMANodeID: 4}, + 77: {CoreID: 77, SocketID: 1, NUMANodeID: 4}, + 78: {CoreID: 78, SocketID: 1, NUMANodeID: 4}, + 79: {CoreID: 79, SocketID: 1, NUMANodeID: 4}, + 80: {CoreID: 80, SocketID: 1, NUMANodeID: 5}, + 81: {CoreID: 81, SocketID: 1, NUMANodeID: 5}, + 82: {CoreID: 82, SocketID: 1, NUMANodeID: 5}, + 83: {CoreID: 83, SocketID: 1, NUMANodeID: 5}, + 84: {CoreID: 84, SocketID: 1, NUMANodeID: 5}, + 85: {CoreID: 85, SocketID: 1, NUMANodeID: 5}, + 86: {CoreID: 86, SocketID: 1, NUMANodeID: 5}, + 87: {CoreID: 87, SocketID: 1, NUMANodeID: 5}, + 88: {CoreID: 88, SocketID: 1, NUMANodeID: 5}, + 89: {CoreID: 89, SocketID: 1, NUMANodeID: 5}, + 90: {CoreID: 90, SocketID: 1, NUMANodeID: 5}, + 91: {CoreID: 91, SocketID: 1, NUMANodeID: 5}, + 92: {CoreID: 92, SocketID: 1, NUMANodeID: 5}, + 93: {CoreID: 93, SocketID: 1, NUMANodeID: 5}, + 94: {CoreID: 94, SocketID: 1, NUMANodeID: 5}, + 95: {CoreID: 95, SocketID: 1, NUMANodeID: 5}, + 96: {CoreID: 96, SocketID: 1, NUMANodeID: 6}, + 97: {CoreID: 97, SocketID: 1, NUMANodeID: 6}, + 98: {CoreID: 98, SocketID: 1, NUMANodeID: 6}, + 99: {CoreID: 99, SocketID: 1, NUMANodeID: 6}, + 100: {CoreID: 100, SocketID: 1, NUMANodeID: 6}, + 101: {CoreID: 101, SocketID: 1, NUMANodeID: 6}, + 102: {CoreID: 102, SocketID: 1, NUMANodeID: 6}, + 103: {CoreID: 103, SocketID: 1, NUMANodeID: 6}, + 104: {CoreID: 104, SocketID: 1, NUMANodeID: 6}, + 105: {CoreID: 105, SocketID: 1, NUMANodeID: 6}, + 106: {CoreID: 106, SocketID: 1, NUMANodeID: 6}, + 107: {CoreID: 107, SocketID: 1, NUMANodeID: 6}, + 108: {CoreID: 108, SocketID: 1, NUMANodeID: 6}, + 109: {CoreID: 109, SocketID: 1, NUMANodeID: 6}, + 110: {CoreID: 110, SocketID: 1, NUMANodeID: 6}, + 111: {CoreID: 111, SocketID: 1, NUMANodeID: 6}, + 112: {CoreID: 112, SocketID: 1, NUMANodeID: 7}, + 113: {CoreID: 113, SocketID: 1, NUMANodeID: 7}, + 114: {CoreID: 114, SocketID: 1, NUMANodeID: 7}, + 115: {CoreID: 115, SocketID: 1, NUMANodeID: 7}, + 116: {CoreID: 116, SocketID: 1, NUMANodeID: 7}, + 117: {CoreID: 117, SocketID: 1, NUMANodeID: 7}, + 118: {CoreID: 118, SocketID: 1, NUMANodeID: 7}, + 119: {CoreID: 119, SocketID: 1, NUMANodeID: 7}, + 120: {CoreID: 120, SocketID: 1, NUMANodeID: 7}, + 121: {CoreID: 121, SocketID: 1, NUMANodeID: 7}, + 122: {CoreID: 122, SocketID: 1, NUMANodeID: 7}, + 123: {CoreID: 123, SocketID: 1, NUMANodeID: 7}, + 124: {CoreID: 124, SocketID: 1, NUMANodeID: 7}, + 125: {CoreID: 125, SocketID: 1, NUMANodeID: 7}, + 126: {CoreID: 126, SocketID: 1, NUMANodeID: 7}, + 127: {CoreID: 127, SocketID: 1, NUMANodeID: 7}, + 128: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 129: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 130: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 131: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 132: {CoreID: 4, SocketID: 0, NUMANodeID: 0}, + 133: {CoreID: 5, SocketID: 0, NUMANodeID: 0}, + 134: {CoreID: 6, SocketID: 0, NUMANodeID: 0}, + 135: {CoreID: 7, SocketID: 0, NUMANodeID: 0}, + 136: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 137: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 138: {CoreID: 10, SocketID: 0, NUMANodeID: 0}, + 139: {CoreID: 11, SocketID: 0, NUMANodeID: 0}, + 140: {CoreID: 12, SocketID: 0, NUMANodeID: 0}, + 141: {CoreID: 13, SocketID: 0, NUMANodeID: 0}, + 142: {CoreID: 14, SocketID: 0, NUMANodeID: 0}, + 143: {CoreID: 15, SocketID: 0, NUMANodeID: 0}, + 144: {CoreID: 16, SocketID: 0, NUMANodeID: 1}, + 145: {CoreID: 17, SocketID: 0, NUMANodeID: 1}, + 146: {CoreID: 18, SocketID: 0, NUMANodeID: 1}, + 147: {CoreID: 19, SocketID: 0, NUMANodeID: 1}, + 148: {CoreID: 20, SocketID: 0, NUMANodeID: 1}, + 149: {CoreID: 21, SocketID: 0, NUMANodeID: 1}, + 150: {CoreID: 22, SocketID: 0, NUMANodeID: 1}, + 151: {CoreID: 23, SocketID: 0, NUMANodeID: 1}, + 152: {CoreID: 24, SocketID: 0, NUMANodeID: 1}, + 153: {CoreID: 25, SocketID: 0, NUMANodeID: 1}, + 154: {CoreID: 26, SocketID: 0, NUMANodeID: 1}, + 155: {CoreID: 27, SocketID: 0, NUMANodeID: 1}, + 156: {CoreID: 28, SocketID: 0, NUMANodeID: 1}, + 157: {CoreID: 29, SocketID: 0, NUMANodeID: 1}, + 158: {CoreID: 30, SocketID: 0, NUMANodeID: 1}, + 159: {CoreID: 31, SocketID: 0, NUMANodeID: 1}, + 160: {CoreID: 32, SocketID: 0, NUMANodeID: 2}, + 161: {CoreID: 33, SocketID: 0, NUMANodeID: 2}, + 162: {CoreID: 34, SocketID: 0, NUMANodeID: 2}, + 163: {CoreID: 35, SocketID: 0, NUMANodeID: 2}, + 164: {CoreID: 36, SocketID: 0, NUMANodeID: 2}, + 165: {CoreID: 37, SocketID: 0, NUMANodeID: 2}, + 166: {CoreID: 38, SocketID: 0, NUMANodeID: 2}, + 167: {CoreID: 39, SocketID: 0, NUMANodeID: 2}, + 168: {CoreID: 40, SocketID: 0, NUMANodeID: 2}, + 169: {CoreID: 41, SocketID: 0, NUMANodeID: 2}, + 170: {CoreID: 42, SocketID: 0, NUMANodeID: 2}, + 171: {CoreID: 43, SocketID: 0, NUMANodeID: 2}, + 172: {CoreID: 44, SocketID: 0, NUMANodeID: 2}, + 173: {CoreID: 45, SocketID: 0, NUMANodeID: 2}, + 174: {CoreID: 46, SocketID: 0, NUMANodeID: 2}, + 175: {CoreID: 47, SocketID: 0, NUMANodeID: 2}, + 176: {CoreID: 48, SocketID: 0, NUMANodeID: 3}, + 177: {CoreID: 49, SocketID: 0, NUMANodeID: 3}, + 178: {CoreID: 50, SocketID: 0, NUMANodeID: 3}, + 179: {CoreID: 51, SocketID: 0, NUMANodeID: 3}, + 180: {CoreID: 52, SocketID: 0, NUMANodeID: 3}, + 181: {CoreID: 53, SocketID: 0, NUMANodeID: 3}, + 182: {CoreID: 54, SocketID: 0, NUMANodeID: 3}, + 183: {CoreID: 55, SocketID: 0, NUMANodeID: 3}, + 184: {CoreID: 56, SocketID: 0, NUMANodeID: 3}, + 185: {CoreID: 57, SocketID: 0, NUMANodeID: 3}, + 186: {CoreID: 58, SocketID: 0, NUMANodeID: 3}, + 187: {CoreID: 59, SocketID: 0, NUMANodeID: 3}, + 188: {CoreID: 60, SocketID: 0, NUMANodeID: 3}, + 189: {CoreID: 61, SocketID: 0, NUMANodeID: 3}, + 190: {CoreID: 62, SocketID: 0, NUMANodeID: 3}, + 191: {CoreID: 63, SocketID: 0, NUMANodeID: 3}, + 192: {CoreID: 64, SocketID: 1, NUMANodeID: 4}, + 193: {CoreID: 65, SocketID: 1, NUMANodeID: 4}, + 194: {CoreID: 66, SocketID: 1, NUMANodeID: 4}, + 195: {CoreID: 67, SocketID: 1, NUMANodeID: 4}, + 196: {CoreID: 68, SocketID: 1, NUMANodeID: 4}, + 197: {CoreID: 69, SocketID: 1, NUMANodeID: 4}, + 198: {CoreID: 70, SocketID: 1, NUMANodeID: 4}, + 199: {CoreID: 71, SocketID: 1, NUMANodeID: 4}, + 200: {CoreID: 72, SocketID: 1, NUMANodeID: 4}, + 201: {CoreID: 73, SocketID: 1, NUMANodeID: 4}, + 202: {CoreID: 74, SocketID: 1, NUMANodeID: 4}, + 203: {CoreID: 75, SocketID: 1, NUMANodeID: 4}, + 204: {CoreID: 76, SocketID: 1, NUMANodeID: 4}, + 205: {CoreID: 77, SocketID: 1, NUMANodeID: 4}, + 206: {CoreID: 78, SocketID: 1, NUMANodeID: 4}, + 207: {CoreID: 79, SocketID: 1, NUMANodeID: 4}, + 208: {CoreID: 80, SocketID: 1, NUMANodeID: 5}, + 209: {CoreID: 81, SocketID: 1, NUMANodeID: 5}, + 210: {CoreID: 82, SocketID: 1, NUMANodeID: 5}, + 211: {CoreID: 83, SocketID: 1, NUMANodeID: 5}, + 212: {CoreID: 84, SocketID: 1, NUMANodeID: 5}, + 213: {CoreID: 85, SocketID: 1, NUMANodeID: 5}, + 214: {CoreID: 86, SocketID: 1, NUMANodeID: 5}, + 215: {CoreID: 87, SocketID: 1, NUMANodeID: 5}, + 216: {CoreID: 88, SocketID: 1, NUMANodeID: 5}, + 217: {CoreID: 89, SocketID: 1, NUMANodeID: 5}, + 218: {CoreID: 90, SocketID: 1, NUMANodeID: 5}, + 219: {CoreID: 91, SocketID: 1, NUMANodeID: 5}, + 220: {CoreID: 92, SocketID: 1, NUMANodeID: 5}, + 221: {CoreID: 93, SocketID: 1, NUMANodeID: 5}, + 222: {CoreID: 94, SocketID: 1, NUMANodeID: 5}, + 223: {CoreID: 95, SocketID: 1, NUMANodeID: 5}, + 224: {CoreID: 96, SocketID: 1, NUMANodeID: 6}, + 225: {CoreID: 97, SocketID: 1, NUMANodeID: 6}, + 226: {CoreID: 98, SocketID: 1, NUMANodeID: 6}, + 227: {CoreID: 99, SocketID: 1, NUMANodeID: 6}, + 228: {CoreID: 100, SocketID: 1, NUMANodeID: 6}, + 229: {CoreID: 101, SocketID: 1, NUMANodeID: 6}, + 230: {CoreID: 102, SocketID: 1, NUMANodeID: 6}, + 231: {CoreID: 103, SocketID: 1, NUMANodeID: 6}, + 232: {CoreID: 104, SocketID: 1, NUMANodeID: 6}, + 233: {CoreID: 105, SocketID: 1, NUMANodeID: 6}, + 234: {CoreID: 106, SocketID: 1, NUMANodeID: 6}, + 235: {CoreID: 107, SocketID: 1, NUMANodeID: 6}, + 236: {CoreID: 108, SocketID: 1, NUMANodeID: 6}, + 237: {CoreID: 109, SocketID: 1, NUMANodeID: 6}, + 238: {CoreID: 110, SocketID: 1, NUMANodeID: 6}, + 239: {CoreID: 111, SocketID: 1, NUMANodeID: 6}, + 240: {CoreID: 112, SocketID: 1, NUMANodeID: 7}, + 241: {CoreID: 113, SocketID: 1, NUMANodeID: 7}, + 242: {CoreID: 114, SocketID: 1, NUMANodeID: 7}, + 243: {CoreID: 115, SocketID: 1, NUMANodeID: 7}, + 244: {CoreID: 116, SocketID: 1, NUMANodeID: 7}, + 245: {CoreID: 117, SocketID: 1, NUMANodeID: 7}, + 246: {CoreID: 118, SocketID: 1, NUMANodeID: 7}, + 247: {CoreID: 119, SocketID: 1, NUMANodeID: 7}, + 248: {CoreID: 120, SocketID: 1, NUMANodeID: 7}, + 249: {CoreID: 121, SocketID: 1, NUMANodeID: 7}, + 250: {CoreID: 122, SocketID: 1, NUMANodeID: 7}, + 251: {CoreID: 123, SocketID: 1, NUMANodeID: 7}, + 252: {CoreID: 124, SocketID: 1, NUMANodeID: 7}, + 253: {CoreID: 125, SocketID: 1, NUMANodeID: 7}, + 254: {CoreID: 126, SocketID: 1, NUMANodeID: 7}, + 255: {CoreID: 127, SocketID: 1, NUMANodeID: 7}, + }, + } )