From ecc14fe661c22f5da967a7ff50cfb3aead60905b Mon Sep 17 00:00:00 2001 From: Kevin Klues Date: Mon, 26 Aug 2019 09:20:05 -0500 Subject: [PATCH] Update CPUManager to include NUMANodeID in CPUTopology Unfortunately, the NUMA information is not readily available from cadvisor, so we have to roll the logic to discover it by hand. In the future, we should remove this custiom code to use the information provided by cadvisor once it is made available. --- pkg/kubelet/cm/BUILD | 1 + pkg/kubelet/cm/container_manager_linux.go | 9 + pkg/kubelet/cm/cpumanager/cpu_manager.go | 4 +- pkg/kubelet/cm/cpumanager/cpu_manager_test.go | 2 +- pkg/kubelet/cm/cpumanager/policy_test.go | 632 +++++++++--------- pkg/kubelet/cm/cpumanager/topology/BUILD | 5 +- .../cm/cpumanager/topology/topology.go | 132 +++- .../cm/cpumanager/topology/topology_test.go | 98 +-- .../cm/cpumanager/topology_hints_test.go | 7 +- 9 files changed, 521 insertions(+), 369 deletions(-) diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 46f1c8fccd4..96aaa1996da 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -69,6 +69,7 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/cm/cpumanager/topology:go_default_library", "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/cm/util:go_default_library", "//pkg/kubelet/events:go_default_library", diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index da651b7842d..5a171cee187 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -47,6 +47,7 @@ import ( podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" + cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util" @@ -226,6 +227,13 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I if err != nil { return nil, err } + // Correct NUMA information is currently missing from cadvisor's + // MachineInfo struct, so we use the CPUManager's internal logic for + // gathering NUMANodeInfo to pass to components that care about it. + numaNodeInfo, err := cputopology.GetNUMANodeInfo() + if err != nil { + return nil, err + } capacity := cadvisor.CapacityFromMachineInfo(machineInfo) for k, v := range capacity { internalCapacity[k] = v @@ -309,6 +317,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I nodeConfig.ExperimentalCPUManagerPolicy, nodeConfig.ExperimentalCPUManagerReconcilePeriod, machineInfo, + numaNodeInfo, cm.GetNodeAllocatableReservation(), nodeConfig.KubeletRootDir, cm.topologyManager, diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go index 37042707607..0efe986ed88 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -102,7 +102,7 @@ type manager struct { var _ Manager = &manager{} // NewManager creates new cpu manager based on provided policy -func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) { +func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, numaNodeInfo topology.NUMANodeInfo, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) { var topo *topology.CPUTopology var policy Policy @@ -113,7 +113,7 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo case PolicyStatic: var err error - topo, err := topology.Discover(machineInfo) + topo, err := topology.Discover(machineInfo, numaNodeInfo) if err != nil { return nil, err } diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 404ceece252..92e5792fe3f 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -343,7 +343,7 @@ func TestCPUManagerGenerate(t *testing.T) { } defer os.RemoveAll(sDir) - mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager()) + mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, nil, testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager()) if testCase.expectedError != nil { if !strings.Contains(err.Error(), testCase.expectedError.Error()) { t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error()) diff --git a/pkg/kubelet/cm/cpumanager/policy_test.go b/pkg/kubelet/cm/cpumanager/policy_test.go index c8d3c6b8276..02e3709c01d 100644 --- a/pkg/kubelet/cm/cpumanager/policy_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_test.go @@ -26,14 +26,14 @@ var ( NumSockets: 1, NumCores: 4, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 0, SocketID: 0}, - 5: {CoreID: 1, SocketID: 0}, - 6: {CoreID: 2, SocketID: 0}, - 7: {CoreID: 3, SocketID: 0}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 6: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, }, } @@ -42,18 +42,18 @@ var ( NumSockets: 2, NumCores: 6, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 1}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 1}, - 4: {CoreID: 4, SocketID: 0}, - 5: {CoreID: 5, SocketID: 1}, - 6: {CoreID: 0, SocketID: 0}, - 7: {CoreID: 1, SocketID: 1}, - 8: {CoreID: 2, SocketID: 0}, - 9: {CoreID: 3, SocketID: 1}, - 10: {CoreID: 4, SocketID: 0}, - 11: {CoreID: 5, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 1, NUMANodeID: 1}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 4: {CoreID: 4, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, + 6: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 1, SocketID: 1, NUMANodeID: 1}, + 8: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 10: {CoreID: 4, SocketID: 0, NUMANodeID: 0}, + 11: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, }, } @@ -62,14 +62,14 @@ var ( NumSockets: 2, NumCores: 8, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 4, SocketID: 1}, - 5: {CoreID: 5, SocketID: 1}, - 6: {CoreID: 6, SocketID: 1}, - 7: {CoreID: 7, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 4, SocketID: 1, NUMANodeID: 1}, + 5: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, + 6: {CoreID: 6, SocketID: 1, NUMANodeID: 1}, + 7: {CoreID: 7, SocketID: 1, NUMANodeID: 1}, }, } @@ -97,294 +97,294 @@ var ( NumSockets: 4, NumCores: 72, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 169: {CoreID: 0, SocketID: 0}, - 109: {CoreID: 0, SocketID: 0}, - 229: {CoreID: 0, SocketID: 0}, - 50: {CoreID: 1, SocketID: 0}, - 170: {CoreID: 1, SocketID: 0}, - 110: {CoreID: 1, SocketID: 0}, - 230: {CoreID: 1, SocketID: 0}, - 1: {CoreID: 64, SocketID: 0}, - 25: {CoreID: 64, SocketID: 0}, - 13: {CoreID: 64, SocketID: 0}, - 38: {CoreID: 64, SocketID: 0}, - 2: {CoreID: 65, SocketID: 0}, - 26: {CoreID: 65, SocketID: 0}, - 14: {CoreID: 65, SocketID: 0}, - 39: {CoreID: 65, SocketID: 0}, - 9: {CoreID: 72, SocketID: 0}, - 33: {CoreID: 72, SocketID: 0}, - 21: {CoreID: 72, SocketID: 0}, - 46: {CoreID: 72, SocketID: 0}, - 10: {CoreID: 73, SocketID: 0}, - 34: {CoreID: 73, SocketID: 0}, - 22: {CoreID: 73, SocketID: 0}, - 47: {CoreID: 73, SocketID: 0}, - 57: {CoreID: 8, SocketID: 0}, - 177: {CoreID: 8, SocketID: 0}, - 117: {CoreID: 8, SocketID: 0}, - 237: {CoreID: 8, SocketID: 0}, - 58: {CoreID: 9, SocketID: 0}, - 178: {CoreID: 9, SocketID: 0}, - 118: {CoreID: 9, SocketID: 0}, - 238: {CoreID: 9, SocketID: 0}, - 71: {CoreID: 24, SocketID: 0}, - 191: {CoreID: 24, SocketID: 0}, - 131: {CoreID: 24, SocketID: 0}, - 251: {CoreID: 24, SocketID: 0}, - 72: {CoreID: 25, SocketID: 0}, - 192: {CoreID: 25, SocketID: 0}, - 132: {CoreID: 25, SocketID: 0}, - 252: {CoreID: 25, SocketID: 0}, - 79: {CoreID: 32, SocketID: 0}, - 199: {CoreID: 32, SocketID: 0}, - 139: {CoreID: 32, SocketID: 0}, - 259: {CoreID: 32, SocketID: 0}, - 80: {CoreID: 33, SocketID: 0}, - 200: {CoreID: 33, SocketID: 0}, - 140: {CoreID: 33, SocketID: 0}, - 260: {CoreID: 33, SocketID: 0}, - 87: {CoreID: 40, SocketID: 0}, - 207: {CoreID: 40, SocketID: 0}, - 147: {CoreID: 40, SocketID: 0}, - 267: {CoreID: 40, SocketID: 0}, - 88: {CoreID: 41, SocketID: 0}, - 208: {CoreID: 41, SocketID: 0}, - 148: {CoreID: 41, SocketID: 0}, - 268: {CoreID: 41, SocketID: 0}, - 95: {CoreID: 48, SocketID: 0}, - 215: {CoreID: 48, SocketID: 0}, - 155: {CoreID: 48, SocketID: 0}, - 275: {CoreID: 48, SocketID: 0}, - 96: {CoreID: 49, SocketID: 0}, - 216: {CoreID: 49, SocketID: 0}, - 156: {CoreID: 49, SocketID: 0}, - 276: {CoreID: 49, SocketID: 0}, - 103: {CoreID: 56, SocketID: 0}, - 223: {CoreID: 56, SocketID: 0}, - 163: {CoreID: 56, SocketID: 0}, - 283: {CoreID: 56, SocketID: 0}, - 104: {CoreID: 57, SocketID: 0}, - 224: {CoreID: 57, SocketID: 0}, - 164: {CoreID: 57, SocketID: 0}, - 284: {CoreID: 57, SocketID: 0}, - 3: {CoreID: 66, SocketID: 1}, - 27: {CoreID: 66, SocketID: 1}, - 15: {CoreID: 66, SocketID: 1}, - 40: {CoreID: 66, SocketID: 1}, - 4: {CoreID: 67, SocketID: 1}, - 28: {CoreID: 67, SocketID: 1}, - 16: {CoreID: 67, SocketID: 1}, - 41: {CoreID: 67, SocketID: 1}, - 11: {CoreID: 74, SocketID: 1}, - 35: {CoreID: 74, SocketID: 1}, - 23: {CoreID: 74, SocketID: 1}, - 48: {CoreID: 74, SocketID: 1}, - 12: {CoreID: 75, SocketID: 1}, - 36: {CoreID: 75, SocketID: 1}, - 24: {CoreID: 75, SocketID: 1}, - 49: {CoreID: 75, SocketID: 1}, - 51: {CoreID: 2, SocketID: 1}, - 171: {CoreID: 2, SocketID: 1}, - 111: {CoreID: 2, SocketID: 1}, - 231: {CoreID: 2, SocketID: 1}, - 52: {CoreID: 3, SocketID: 1}, - 172: {CoreID: 3, SocketID: 1}, - 112: {CoreID: 3, SocketID: 1}, - 232: {CoreID: 3, SocketID: 1}, - 59: {CoreID: 10, SocketID: 1}, - 179: {CoreID: 10, SocketID: 1}, - 119: {CoreID: 10, SocketID: 1}, - 239: {CoreID: 10, SocketID: 1}, - 60: {CoreID: 11, SocketID: 1}, - 180: {CoreID: 11, SocketID: 1}, - 120: {CoreID: 11, SocketID: 1}, - 240: {CoreID: 11, SocketID: 1}, - 65: {CoreID: 18, SocketID: 1}, - 185: {CoreID: 18, SocketID: 1}, - 125: {CoreID: 18, SocketID: 1}, - 245: {CoreID: 18, SocketID: 1}, - 66: {CoreID: 19, SocketID: 1}, - 186: {CoreID: 19, SocketID: 1}, - 126: {CoreID: 19, SocketID: 1}, - 246: {CoreID: 19, SocketID: 1}, - 73: {CoreID: 26, SocketID: 1}, - 193: {CoreID: 26, SocketID: 1}, - 133: {CoreID: 26, SocketID: 1}, - 253: {CoreID: 26, SocketID: 1}, - 74: {CoreID: 27, SocketID: 1}, - 194: {CoreID: 27, SocketID: 1}, - 134: {CoreID: 27, SocketID: 1}, - 254: {CoreID: 27, SocketID: 1}, - 81: {CoreID: 34, SocketID: 1}, - 201: {CoreID: 34, SocketID: 1}, - 141: {CoreID: 34, SocketID: 1}, - 261: {CoreID: 34, SocketID: 1}, - 82: {CoreID: 35, SocketID: 1}, - 202: {CoreID: 35, SocketID: 1}, - 142: {CoreID: 35, SocketID: 1}, - 262: {CoreID: 35, SocketID: 1}, - 89: {CoreID: 42, SocketID: 1}, - 209: {CoreID: 42, SocketID: 1}, - 149: {CoreID: 42, SocketID: 1}, - 269: {CoreID: 42, SocketID: 1}, - 90: {CoreID: 43, SocketID: 1}, - 210: {CoreID: 43, SocketID: 1}, - 150: {CoreID: 43, SocketID: 1}, - 270: {CoreID: 43, SocketID: 1}, - 97: {CoreID: 50, SocketID: 1}, - 217: {CoreID: 50, SocketID: 1}, - 157: {CoreID: 50, SocketID: 1}, - 277: {CoreID: 50, SocketID: 1}, - 98: {CoreID: 51, SocketID: 1}, - 218: {CoreID: 51, SocketID: 1}, - 158: {CoreID: 51, SocketID: 1}, - 278: {CoreID: 51, SocketID: 1}, - 5: {CoreID: 68, SocketID: 2}, - 29: {CoreID: 68, SocketID: 2}, - 17: {CoreID: 68, SocketID: 2}, - 42: {CoreID: 68, SocketID: 2}, - 6: {CoreID: 69, SocketID: 2}, - 30: {CoreID: 69, SocketID: 2}, - 18: {CoreID: 69, SocketID: 2}, - 43: {CoreID: 69, SocketID: 2}, - 53: {CoreID: 4, SocketID: 2}, - 173: {CoreID: 4, SocketID: 2}, - 113: {CoreID: 4, SocketID: 2}, - 233: {CoreID: 4, SocketID: 2}, - 54: {CoreID: 5, SocketID: 2}, - 174: {CoreID: 5, SocketID: 2}, - 114: {CoreID: 5, SocketID: 2}, - 234: {CoreID: 5, SocketID: 2}, - 61: {CoreID: 12, SocketID: 2}, - 181: {CoreID: 12, SocketID: 2}, - 121: {CoreID: 12, SocketID: 2}, - 241: {CoreID: 12, SocketID: 2}, - 62: {CoreID: 13, SocketID: 2}, - 182: {CoreID: 13, SocketID: 2}, - 122: {CoreID: 13, SocketID: 2}, - 242: {CoreID: 13, SocketID: 2}, - 67: {CoreID: 20, SocketID: 2}, - 187: {CoreID: 20, SocketID: 2}, - 127: {CoreID: 20, SocketID: 2}, - 247: {CoreID: 20, SocketID: 2}, - 68: {CoreID: 21, SocketID: 2}, - 188: {CoreID: 21, SocketID: 2}, - 128: {CoreID: 21, SocketID: 2}, - 248: {CoreID: 21, SocketID: 2}, - 75: {CoreID: 28, SocketID: 2}, - 195: {CoreID: 28, SocketID: 2}, - 135: {CoreID: 28, SocketID: 2}, - 255: {CoreID: 28, SocketID: 2}, - 76: {CoreID: 29, SocketID: 2}, - 196: {CoreID: 29, SocketID: 2}, - 136: {CoreID: 29, SocketID: 2}, - 256: {CoreID: 29, SocketID: 2}, - 83: {CoreID: 36, SocketID: 2}, - 203: {CoreID: 36, SocketID: 2}, - 143: {CoreID: 36, SocketID: 2}, - 263: {CoreID: 36, SocketID: 2}, - 84: {CoreID: 37, SocketID: 2}, - 204: {CoreID: 37, SocketID: 2}, - 144: {CoreID: 37, SocketID: 2}, - 264: {CoreID: 37, SocketID: 2}, - 91: {CoreID: 44, SocketID: 2}, - 211: {CoreID: 44, SocketID: 2}, - 151: {CoreID: 44, SocketID: 2}, - 271: {CoreID: 44, SocketID: 2}, - 92: {CoreID: 45, SocketID: 2}, - 212: {CoreID: 45, SocketID: 2}, - 152: {CoreID: 45, SocketID: 2}, - 272: {CoreID: 45, SocketID: 2}, - 99: {CoreID: 52, SocketID: 2}, - 219: {CoreID: 52, SocketID: 2}, - 159: {CoreID: 52, SocketID: 2}, - 279: {CoreID: 52, SocketID: 2}, - 100: {CoreID: 53, SocketID: 2}, - 220: {CoreID: 53, SocketID: 2}, - 160: {CoreID: 53, SocketID: 2}, - 280: {CoreID: 53, SocketID: 2}, - 105: {CoreID: 60, SocketID: 2}, - 225: {CoreID: 60, SocketID: 2}, - 165: {CoreID: 60, SocketID: 2}, - 285: {CoreID: 60, SocketID: 2}, - 106: {CoreID: 61, SocketID: 2}, - 226: {CoreID: 61, SocketID: 2}, - 166: {CoreID: 61, SocketID: 2}, - 286: {CoreID: 61, SocketID: 2}, - 7: {CoreID: 70, SocketID: 3}, - 31: {CoreID: 70, SocketID: 3}, - 19: {CoreID: 70, SocketID: 3}, - 44: {CoreID: 70, SocketID: 3}, - 8: {CoreID: 71, SocketID: 3}, - 32: {CoreID: 71, SocketID: 3}, - 20: {CoreID: 71, SocketID: 3}, - 45: {CoreID: 71, SocketID: 3}, - 37: {CoreID: 63, SocketID: 3}, - 168: {CoreID: 63, SocketID: 3}, - 108: {CoreID: 63, SocketID: 3}, - 228: {CoreID: 63, SocketID: 3}, - 107: {CoreID: 62, SocketID: 3}, - 227: {CoreID: 62, SocketID: 3}, - 167: {CoreID: 62, SocketID: 3}, - 287: {CoreID: 62, SocketID: 3}, - 55: {CoreID: 6, SocketID: 3}, - 175: {CoreID: 6, SocketID: 3}, - 115: {CoreID: 6, SocketID: 3}, - 235: {CoreID: 6, SocketID: 3}, - 56: {CoreID: 7, SocketID: 3}, - 176: {CoreID: 7, SocketID: 3}, - 116: {CoreID: 7, SocketID: 3}, - 236: {CoreID: 7, SocketID: 3}, - 63: {CoreID: 14, SocketID: 3}, - 183: {CoreID: 14, SocketID: 3}, - 123: {CoreID: 14, SocketID: 3}, - 243: {CoreID: 14, SocketID: 3}, - 64: {CoreID: 15, SocketID: 3}, - 184: {CoreID: 15, SocketID: 3}, - 124: {CoreID: 15, SocketID: 3}, - 244: {CoreID: 15, SocketID: 3}, - 69: {CoreID: 22, SocketID: 3}, - 189: {CoreID: 22, SocketID: 3}, - 129: {CoreID: 22, SocketID: 3}, - 249: {CoreID: 22, SocketID: 3}, - 70: {CoreID: 23, SocketID: 3}, - 190: {CoreID: 23, SocketID: 3}, - 130: {CoreID: 23, SocketID: 3}, - 250: {CoreID: 23, SocketID: 3}, - 77: {CoreID: 30, SocketID: 3}, - 197: {CoreID: 30, SocketID: 3}, - 137: {CoreID: 30, SocketID: 3}, - 257: {CoreID: 30, SocketID: 3}, - 78: {CoreID: 31, SocketID: 3}, - 198: {CoreID: 31, SocketID: 3}, - 138: {CoreID: 31, SocketID: 3}, - 258: {CoreID: 31, SocketID: 3}, - 85: {CoreID: 38, SocketID: 3}, - 205: {CoreID: 38, SocketID: 3}, - 145: {CoreID: 38, SocketID: 3}, - 265: {CoreID: 38, SocketID: 3}, - 86: {CoreID: 39, SocketID: 3}, - 206: {CoreID: 39, SocketID: 3}, - 146: {CoreID: 39, SocketID: 3}, - 266: {CoreID: 39, SocketID: 3}, - 93: {CoreID: 46, SocketID: 3}, - 213: {CoreID: 46, SocketID: 3}, - 153: {CoreID: 46, SocketID: 3}, - 273: {CoreID: 46, SocketID: 3}, - 94: {CoreID: 47, SocketID: 3}, - 214: {CoreID: 47, SocketID: 3}, - 154: {CoreID: 47, SocketID: 3}, - 274: {CoreID: 47, SocketID: 3}, - 101: {CoreID: 54, SocketID: 3}, - 221: {CoreID: 54, SocketID: 3}, - 161: {CoreID: 54, SocketID: 3}, - 281: {CoreID: 54, SocketID: 3}, - 102: {CoreID: 55, SocketID: 3}, - 222: {CoreID: 55, SocketID: 3}, - 162: {CoreID: 55, SocketID: 3}, - 282: {CoreID: 55, SocketID: 3}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 169: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 109: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 229: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 50: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 170: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 110: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 230: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 25: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 13: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 38: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 26: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 14: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 39: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 33: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 21: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 46: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 10: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 34: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 22: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 47: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 57: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 177: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 117: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 237: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 58: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 178: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 118: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 238: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 71: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 191: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 131: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 251: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 72: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 192: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 132: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 252: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 79: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 199: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 139: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 259: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 80: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 200: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 140: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 260: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 87: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 207: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 147: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 267: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 88: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 208: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 148: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 268: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 95: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 215: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 155: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 275: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 96: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 216: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 156: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 276: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 103: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 223: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 163: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 283: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 104: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 224: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 164: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 284: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 27: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 15: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 40: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 4: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 28: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 16: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 41: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 11: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 35: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 23: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 48: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 12: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 36: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 24: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 49: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 51: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 171: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 111: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 231: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 52: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 172: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 112: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 232: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 59: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 179: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 119: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 239: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 60: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 180: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 120: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 240: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 65: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 185: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 125: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 245: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 66: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 186: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 126: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 246: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 73: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 193: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 133: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 253: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 74: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 194: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 134: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 254: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 81: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 201: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 141: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 261: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 82: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 202: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 142: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 262: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 89: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 209: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 149: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 269: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 90: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 210: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 150: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 270: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 97: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 217: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 157: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 277: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 98: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 218: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 158: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 278: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 5: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 29: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 17: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 42: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 6: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 30: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 18: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 43: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 53: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 173: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 113: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 233: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 54: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 174: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 114: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 234: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 61: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 181: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 121: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 241: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 62: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 182: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 122: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 242: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 67: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 187: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 127: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 247: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 68: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 188: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 128: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 248: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 75: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 195: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 135: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 255: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 76: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 196: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 136: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 256: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 83: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 203: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 143: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 263: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 84: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 204: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 144: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 264: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 91: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 211: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 151: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 271: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 92: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 212: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 152: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 272: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 99: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 219: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 159: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 279: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 100: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 220: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 160: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 280: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 105: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 225: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 165: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 285: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 106: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 226: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 166: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 286: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 7: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 31: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 19: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 44: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 8: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 32: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 20: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 45: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 37: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 168: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 108: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 228: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 107: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 227: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 167: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 287: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 55: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 175: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 115: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 235: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 56: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 176: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 116: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 236: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 63: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 183: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 123: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 243: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 64: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 184: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 124: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 244: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 69: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 189: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 129: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 249: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 70: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 190: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 130: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 250: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 77: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 197: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 137: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 257: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 78: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 198: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 138: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 258: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 85: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 205: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 145: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 265: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 86: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 206: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 146: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 266: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 93: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 213: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 153: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 273: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 94: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 214: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 154: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 274: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 101: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 221: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 161: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 281: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 102: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, + 222: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, + 162: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, + 282: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, }, } ) diff --git a/pkg/kubelet/cm/cpumanager/topology/BUILD b/pkg/kubelet/cm/cpumanager/topology/BUILD index 18ab41a33f4..bd5fb095200 100644 --- a/pkg/kubelet/cm/cpumanager/topology/BUILD +++ b/pkg/kubelet/cm/cpumanager/topology/BUILD @@ -33,5 +33,8 @@ go_test( name = "go_default_test", srcs = ["topology_test.go"], embed = [":go_default_library"], - deps = ["//vendor/github.com/google/cadvisor/info/v1:go_default_library"], + deps = [ + "//pkg/kubelet/cm/cpuset:go_default_library", + "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + ], ) diff --git a/pkg/kubelet/cm/cpumanager/topology/topology.go b/pkg/kubelet/cm/cpumanager/topology/topology.go index f19acac5fcb..abde76931bf 100644 --- a/pkg/kubelet/cm/cpumanager/topology/topology.go +++ b/pkg/kubelet/cm/cpumanager/topology/topology.go @@ -18,13 +18,19 @@ package topology import ( "fmt" + "io/ioutil" + "strings" cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) -// CPUDetails is a map from CPU ID to Core ID and Socket ID. +// NUMANodeInfo is a map from NUMANode ID to a list of CPU IDs associated with +// that NUMANode. +type NUMANodeInfo map[int]cpuset.CPUSet + +// CPUDetails is a map from CPU ID to Core ID, Socket ID, and NUMA ID. type CPUDetails map[int]CPUInfo // CPUTopology contains details of node cpu, where : @@ -56,10 +62,11 @@ func (topo *CPUTopology) CPUsPerSocket() int { return topo.NumCPUs / topo.NumSockets } -// CPUInfo contains the socket and core IDs associated with a CPU. +// CPUInfo contains the NUMA, socket, and core IDs associated with a CPU. type CPUInfo struct { - SocketID int - CoreID int + NUMANodeID int + SocketID int + CoreID int } // KeepOnly returns a new CPUDetails object with only the supplied cpus. @@ -73,6 +80,28 @@ func (d CPUDetails) KeepOnly(cpus cpuset.CPUSet) CPUDetails { return result } +// NUMANodes returns all of the NUMANode IDs associated with the CPUs in this +// CPUDetails. +func (d CPUDetails) NUMANodes() cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + b.Add(info.NUMANodeID) + } + return b.Result() +} + +// NUMANodesInSocket returns all of the logical NUMANode IDs associated with +// the given Socket ID in this CPUDetails. +func (d CPUDetails) NUMANodesInSocket(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + if info.SocketID == id { + b.Add(info.NUMANodeID) + } + } + return b.Result() +} + // Sockets returns all of the socket IDs associated with the CPUs in this // CPUDetails. func (d CPUDetails) Sockets() cpuset.CPUSet { @@ -95,6 +124,18 @@ func (d CPUDetails) CPUsInSocket(id int) cpuset.CPUSet { return b.Result() } +// SocketsInNUMANode returns all of the logical Socket IDs associated with the +// given NUMANode ID in this CPUDetails. +func (d CPUDetails) SocketsInNUMANode(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + if info.NUMANodeID == id { + b.Add(info.SocketID) + } + } + return b.Result() +} + // Cores returns all of the core IDs associated with the CPUs in this // CPUDetails. func (d CPUDetails) Cores() cpuset.CPUSet { @@ -105,6 +146,18 @@ func (d CPUDetails) Cores() cpuset.CPUSet { return b.Result() } +// CoresInNUMANode returns all of the core IDs associated with the given +// NUMA ID in this CPUDetails. +func (d CPUDetails) CoresInNUMANode(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + if info.NUMANodeID == id { + b.Add(info.CoreID) + } + } + return b.Result() +} + // CoresInSocket returns all of the core IDs associated with the given // socket ID in this CPUDetails. func (d CPUDetails) CoresInSocket(id int) cpuset.CPUSet { @@ -126,6 +179,18 @@ func (d CPUDetails) CPUs() cpuset.CPUSet { return b.Result() } +// CPUsInNUMANode returns all of the logical CPU IDs associated with the given +// NUMANode ID in this CPUDetails. +func (d CPUDetails) CPUsInNUMANode(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for cpu, info := range d { + if info.NUMANodeID == id { + b.Add(cpu) + } + } + return b.Result() +} + // CPUsInCore returns all of the logical CPU IDs associated with the // given core ID in this CPUDetails. func (d CPUDetails) CPUsInCore(id int) cpuset.CPUSet { @@ -139,7 +204,7 @@ func (d CPUDetails) CPUsInCore(id int) cpuset.CPUSet { } // Discover returns CPUTopology based on cadvisor node info -func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) { +func Discover(machineInfo *cadvisorapi.MachineInfo, numaNodeInfo NUMANodeInfo) (*CPUTopology, error) { if machineInfo.NumCores == 0 { return nil, fmt.Errorf("could not detect number of cpus") } @@ -152,9 +217,16 @@ func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) { for _, core := range socket.Cores { if coreID, err := getUniqueCoreID(core.Threads); err == nil { for _, cpu := range core.Threads { + numaNodeID := 0 + for id, cset := range numaNodeInfo { + if cset.Contains(cpu) { + numaNodeID = id + } + } CPUDetails[cpu] = CPUInfo{ - CoreID: coreID, - SocketID: socket.Id, + CoreID: coreID, + SocketID: socket.Id, + NUMANodeID: numaNodeID, } } } else { @@ -194,3 +266,49 @@ func getUniqueCoreID(threads []int) (coreID int, err error) { return min, nil } + +// GetNUMANodeInfo uses sysfs to return a map of NUMANode id to the list of +// CPUs associated with that NUMANode. +// +// TODO: This is a temporary workaround until cadvisor provides this +// information directly in machineInfo. We should remove this once this +// information is available from cadvisor. +func GetNUMANodeInfo() (NUMANodeInfo, error) { + // Get the possible NUMA nodes on this machine. If reading this file + // is not possible, this is not an error. Instead, we just return a + // nil NUMANodeInfo, indicating that no NUMA information is available + // on this machine. This should implicitly be interpreted as having a + // single NUMA node with id 0 for all CPUs. + nodelist, err := ioutil.ReadFile("/sys/devices/system/node/possible") + if err != nil { + return nil, nil + } + + // Parse the nodelist into a set of Node IDs + nodes, err := cpuset.Parse(strings.TrimSpace(string(nodelist))) + if err != nil { + return nil, err + } + + info := make(NUMANodeInfo) + + // For each node... + for _, node := range nodes.ToSlice() { + // Read the 'cpulist' of the NUMA node from sysfs. + path := fmt.Sprintf("/sys/devices/system/node/node%d/cpulist", node) + cpulist, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + // Convert the 'cpulist' into a set of CPUs. + cpus, err := cpuset.Parse(strings.TrimSpace(string(cpulist))) + if err != nil { + return nil, err + } + + info[node] = cpus + } + + return info, nil +} diff --git a/pkg/kubelet/cm/cpumanager/topology/topology_test.go b/pkg/kubelet/cm/cpumanager/topology/topology_test.go index c8441ed245d..59be5e02861 100644 --- a/pkg/kubelet/cm/cpumanager/topology/topology_test.go +++ b/pkg/kubelet/cm/cpumanager/topology/topology_test.go @@ -21,27 +21,30 @@ import ( "testing" cadvisorapi "github.com/google/cadvisor/info/v1" + "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) func Test_Discover(t *testing.T) { tests := []struct { - name string - args *cadvisorapi.MachineInfo - want *CPUTopology - wantErr bool + name string + machineInfo cadvisorapi.MachineInfo + numaNodeInfo NUMANodeInfo + want *CPUTopology + wantErr bool }{ { name: "FailNumCores", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 0, }, - want: &CPUTopology{}, - wantErr: true, + numaNodeInfo: NUMANodeInfo{}, + want: &CPUTopology{}, + wantErr: true, }, { name: "OneSocketHT", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 8, Topology: []cadvisorapi.Node{ {Id: 0, @@ -54,26 +57,29 @@ func Test_Discover(t *testing.T) { }, }, }, + numaNodeInfo: NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7), + }, want: &CPUTopology{ NumCPUs: 8, NumSockets: 1, NumCores: 4, CPUDetails: map[int]CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 0, SocketID: 0}, - 5: {CoreID: 1, SocketID: 0}, - 6: {CoreID: 2, SocketID: 0}, - 7: {CoreID: 3, SocketID: 0}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 6: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, }, }, wantErr: false, }, { name: "DualSocketNoHT", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 4, Topology: []cadvisorapi.Node{ {Id: 0, @@ -90,22 +96,26 @@ func Test_Discover(t *testing.T) { }, }, }, + numaNodeInfo: NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 2), + 1: cpuset.NewCPUSet(1, 3), + }, want: &CPUTopology{ NumCPUs: 4, NumSockets: 2, NumCores: 4, CPUDetails: map[int]CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 1}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 1, NUMANodeID: 1}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, }, }, wantErr: false, }, { name: "DualSocketHT - non unique Core'ID's", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 12, Topology: []cadvisorapi.Node{ {Id: 0, @@ -124,30 +134,34 @@ func Test_Discover(t *testing.T) { }, }, }, + numaNodeInfo: NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8), + 1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11), + }, want: &CPUTopology{ NumCPUs: 12, NumSockets: 2, NumCores: 6, CPUDetails: map[int]CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 1}, - 4: {CoreID: 4, SocketID: 1}, - 5: {CoreID: 5, SocketID: 1}, - 6: {CoreID: 0, SocketID: 0}, - 7: {CoreID: 1, SocketID: 0}, - 8: {CoreID: 2, SocketID: 0}, - 9: {CoreID: 3, SocketID: 1}, - 10: {CoreID: 4, SocketID: 1}, - 11: {CoreID: 5, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 4: {CoreID: 4, SocketID: 1, NUMANodeID: 1}, + 5: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, + 6: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 8: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 10: {CoreID: 4, SocketID: 1, NUMANodeID: 1}, + 11: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, }, }, wantErr: false, }, { name: "OneSocketHT fail", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 8, Topology: []cadvisorapi.Node{ {Id: 0, @@ -160,12 +174,13 @@ func Test_Discover(t *testing.T) { }, }, }, - want: &CPUTopology{}, - wantErr: true, + numaNodeInfo: NUMANodeInfo{}, + want: &CPUTopology{}, + wantErr: true, }, { name: "OneSocketHT fail", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 8, Topology: []cadvisorapi.Node{ {Id: 0, @@ -178,13 +193,14 @@ func Test_Discover(t *testing.T) { }, }, }, - want: &CPUTopology{}, - wantErr: true, + numaNodeInfo: NUMANodeInfo{}, + want: &CPUTopology{}, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := Discover(tt.args) + got, err := Discover(&tt.machineInfo, tt.numaNodeInfo) if err != nil { if tt.wantErr { t.Logf("Discover() expected error = %v", err) diff --git a/pkg/kubelet/cm/cpumanager/topology_hints_test.go b/pkg/kubelet/cm/cpumanager/topology_hints_test.go index 8b340f3f007..8b639dd6000 100644 --- a/pkg/kubelet/cm/cpumanager/topology_hints_test.go +++ b/pkg/kubelet/cm/cpumanager/topology_hints_test.go @@ -70,7 +70,12 @@ func TestGetTopologyHints(t *testing.T) { }, } - topology, _ := topology.Discover(&machineInfo) + numaNodeInfo := topology.NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8), + 1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11), + } + + topology, _ := topology.Discover(&machineInfo, numaNodeInfo) m := manager{ policy: &staticPolicy{