diff --git a/pkg/kubelet/cm/BUILD b/pkg/kubelet/cm/BUILD index 46f1c8fccd4..96aaa1996da 100644 --- a/pkg/kubelet/cm/BUILD +++ b/pkg/kubelet/cm/BUILD @@ -69,6 +69,7 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", + "//pkg/kubelet/cm/cpumanager/topology:go_default_library", "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/cm/util:go_default_library", "//pkg/kubelet/events:go_default_library", diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go index da651b7842d..5a171cee187 100644 --- a/pkg/kubelet/cm/container_manager_linux.go +++ b/pkg/kubelet/cm/container_manager_linux.go @@ -47,6 +47,7 @@ import ( podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" + cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util" @@ -226,6 +227,13 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I if err != nil { return nil, err } + // Correct NUMA information is currently missing from cadvisor's + // MachineInfo struct, so we use the CPUManager's internal logic for + // gathering NUMANodeInfo to pass to components that care about it. + numaNodeInfo, err := cputopology.GetNUMANodeInfo() + if err != nil { + return nil, err + } capacity := cadvisor.CapacityFromMachineInfo(machineInfo) for k, v := range capacity { internalCapacity[k] = v @@ -309,6 +317,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I nodeConfig.ExperimentalCPUManagerPolicy, nodeConfig.ExperimentalCPUManagerReconcilePeriod, machineInfo, + numaNodeInfo, cm.GetNodeAllocatableReservation(), nodeConfig.KubeletRootDir, cm.topologyManager, diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager.go b/pkg/kubelet/cm/cpumanager/cpu_manager.go index 37042707607..0efe986ed88 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -102,7 +102,7 @@ type manager struct { var _ Manager = &manager{} // NewManager creates new cpu manager based on provided policy -func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) { +func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, numaNodeInfo topology.NUMANodeInfo, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) { var topo *topology.CPUTopology var policy Policy @@ -113,7 +113,7 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo case PolicyStatic: var err error - topo, err := topology.Discover(machineInfo) + topo, err := topology.Discover(machineInfo, numaNodeInfo) if err != nil { return nil, err } diff --git a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index 404ceece252..92e5792fe3f 100644 --- a/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -343,7 +343,7 @@ func TestCPUManagerGenerate(t *testing.T) { } defer os.RemoveAll(sDir) - mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager()) + mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, nil, testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager()) if testCase.expectedError != nil { if !strings.Contains(err.Error(), testCase.expectedError.Error()) { t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error()) diff --git a/pkg/kubelet/cm/cpumanager/policy_test.go b/pkg/kubelet/cm/cpumanager/policy_test.go index c8d3c6b8276..02e3709c01d 100644 --- a/pkg/kubelet/cm/cpumanager/policy_test.go +++ b/pkg/kubelet/cm/cpumanager/policy_test.go @@ -26,14 +26,14 @@ var ( NumSockets: 1, NumCores: 4, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 0, SocketID: 0}, - 5: {CoreID: 1, SocketID: 0}, - 6: {CoreID: 2, SocketID: 0}, - 7: {CoreID: 3, SocketID: 0}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 6: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, }, } @@ -42,18 +42,18 @@ var ( NumSockets: 2, NumCores: 6, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 1}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 1}, - 4: {CoreID: 4, SocketID: 0}, - 5: {CoreID: 5, SocketID: 1}, - 6: {CoreID: 0, SocketID: 0}, - 7: {CoreID: 1, SocketID: 1}, - 8: {CoreID: 2, SocketID: 0}, - 9: {CoreID: 3, SocketID: 1}, - 10: {CoreID: 4, SocketID: 0}, - 11: {CoreID: 5, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 1, NUMANodeID: 1}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 4: {CoreID: 4, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, + 6: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 1, SocketID: 1, NUMANodeID: 1}, + 8: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 10: {CoreID: 4, SocketID: 0, NUMANodeID: 0}, + 11: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, }, } @@ -62,14 +62,14 @@ var ( NumSockets: 2, NumCores: 8, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 4, SocketID: 1}, - 5: {CoreID: 5, SocketID: 1}, - 6: {CoreID: 6, SocketID: 1}, - 7: {CoreID: 7, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 4, SocketID: 1, NUMANodeID: 1}, + 5: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, + 6: {CoreID: 6, SocketID: 1, NUMANodeID: 1}, + 7: {CoreID: 7, SocketID: 1, NUMANodeID: 1}, }, } @@ -97,294 +97,294 @@ var ( NumSockets: 4, NumCores: 72, CPUDetails: map[int]topology.CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 169: {CoreID: 0, SocketID: 0}, - 109: {CoreID: 0, SocketID: 0}, - 229: {CoreID: 0, SocketID: 0}, - 50: {CoreID: 1, SocketID: 0}, - 170: {CoreID: 1, SocketID: 0}, - 110: {CoreID: 1, SocketID: 0}, - 230: {CoreID: 1, SocketID: 0}, - 1: {CoreID: 64, SocketID: 0}, - 25: {CoreID: 64, SocketID: 0}, - 13: {CoreID: 64, SocketID: 0}, - 38: {CoreID: 64, SocketID: 0}, - 2: {CoreID: 65, SocketID: 0}, - 26: {CoreID: 65, SocketID: 0}, - 14: {CoreID: 65, SocketID: 0}, - 39: {CoreID: 65, SocketID: 0}, - 9: {CoreID: 72, SocketID: 0}, - 33: {CoreID: 72, SocketID: 0}, - 21: {CoreID: 72, SocketID: 0}, - 46: {CoreID: 72, SocketID: 0}, - 10: {CoreID: 73, SocketID: 0}, - 34: {CoreID: 73, SocketID: 0}, - 22: {CoreID: 73, SocketID: 0}, - 47: {CoreID: 73, SocketID: 0}, - 57: {CoreID: 8, SocketID: 0}, - 177: {CoreID: 8, SocketID: 0}, - 117: {CoreID: 8, SocketID: 0}, - 237: {CoreID: 8, SocketID: 0}, - 58: {CoreID: 9, SocketID: 0}, - 178: {CoreID: 9, SocketID: 0}, - 118: {CoreID: 9, SocketID: 0}, - 238: {CoreID: 9, SocketID: 0}, - 71: {CoreID: 24, SocketID: 0}, - 191: {CoreID: 24, SocketID: 0}, - 131: {CoreID: 24, SocketID: 0}, - 251: {CoreID: 24, SocketID: 0}, - 72: {CoreID: 25, SocketID: 0}, - 192: {CoreID: 25, SocketID: 0}, - 132: {CoreID: 25, SocketID: 0}, - 252: {CoreID: 25, SocketID: 0}, - 79: {CoreID: 32, SocketID: 0}, - 199: {CoreID: 32, SocketID: 0}, - 139: {CoreID: 32, SocketID: 0}, - 259: {CoreID: 32, SocketID: 0}, - 80: {CoreID: 33, SocketID: 0}, - 200: {CoreID: 33, SocketID: 0}, - 140: {CoreID: 33, SocketID: 0}, - 260: {CoreID: 33, SocketID: 0}, - 87: {CoreID: 40, SocketID: 0}, - 207: {CoreID: 40, SocketID: 0}, - 147: {CoreID: 40, SocketID: 0}, - 267: {CoreID: 40, SocketID: 0}, - 88: {CoreID: 41, SocketID: 0}, - 208: {CoreID: 41, SocketID: 0}, - 148: {CoreID: 41, SocketID: 0}, - 268: {CoreID: 41, SocketID: 0}, - 95: {CoreID: 48, SocketID: 0}, - 215: {CoreID: 48, SocketID: 0}, - 155: {CoreID: 48, SocketID: 0}, - 275: {CoreID: 48, SocketID: 0}, - 96: {CoreID: 49, SocketID: 0}, - 216: {CoreID: 49, SocketID: 0}, - 156: {CoreID: 49, SocketID: 0}, - 276: {CoreID: 49, SocketID: 0}, - 103: {CoreID: 56, SocketID: 0}, - 223: {CoreID: 56, SocketID: 0}, - 163: {CoreID: 56, SocketID: 0}, - 283: {CoreID: 56, SocketID: 0}, - 104: {CoreID: 57, SocketID: 0}, - 224: {CoreID: 57, SocketID: 0}, - 164: {CoreID: 57, SocketID: 0}, - 284: {CoreID: 57, SocketID: 0}, - 3: {CoreID: 66, SocketID: 1}, - 27: {CoreID: 66, SocketID: 1}, - 15: {CoreID: 66, SocketID: 1}, - 40: {CoreID: 66, SocketID: 1}, - 4: {CoreID: 67, SocketID: 1}, - 28: {CoreID: 67, SocketID: 1}, - 16: {CoreID: 67, SocketID: 1}, - 41: {CoreID: 67, SocketID: 1}, - 11: {CoreID: 74, SocketID: 1}, - 35: {CoreID: 74, SocketID: 1}, - 23: {CoreID: 74, SocketID: 1}, - 48: {CoreID: 74, SocketID: 1}, - 12: {CoreID: 75, SocketID: 1}, - 36: {CoreID: 75, SocketID: 1}, - 24: {CoreID: 75, SocketID: 1}, - 49: {CoreID: 75, SocketID: 1}, - 51: {CoreID: 2, SocketID: 1}, - 171: {CoreID: 2, SocketID: 1}, - 111: {CoreID: 2, SocketID: 1}, - 231: {CoreID: 2, SocketID: 1}, - 52: {CoreID: 3, SocketID: 1}, - 172: {CoreID: 3, SocketID: 1}, - 112: {CoreID: 3, SocketID: 1}, - 232: {CoreID: 3, SocketID: 1}, - 59: {CoreID: 10, SocketID: 1}, - 179: {CoreID: 10, SocketID: 1}, - 119: {CoreID: 10, SocketID: 1}, - 239: {CoreID: 10, SocketID: 1}, - 60: {CoreID: 11, SocketID: 1}, - 180: {CoreID: 11, SocketID: 1}, - 120: {CoreID: 11, SocketID: 1}, - 240: {CoreID: 11, SocketID: 1}, - 65: {CoreID: 18, SocketID: 1}, - 185: {CoreID: 18, SocketID: 1}, - 125: {CoreID: 18, SocketID: 1}, - 245: {CoreID: 18, SocketID: 1}, - 66: {CoreID: 19, SocketID: 1}, - 186: {CoreID: 19, SocketID: 1}, - 126: {CoreID: 19, SocketID: 1}, - 246: {CoreID: 19, SocketID: 1}, - 73: {CoreID: 26, SocketID: 1}, - 193: {CoreID: 26, SocketID: 1}, - 133: {CoreID: 26, SocketID: 1}, - 253: {CoreID: 26, SocketID: 1}, - 74: {CoreID: 27, SocketID: 1}, - 194: {CoreID: 27, SocketID: 1}, - 134: {CoreID: 27, SocketID: 1}, - 254: {CoreID: 27, SocketID: 1}, - 81: {CoreID: 34, SocketID: 1}, - 201: {CoreID: 34, SocketID: 1}, - 141: {CoreID: 34, SocketID: 1}, - 261: {CoreID: 34, SocketID: 1}, - 82: {CoreID: 35, SocketID: 1}, - 202: {CoreID: 35, SocketID: 1}, - 142: {CoreID: 35, SocketID: 1}, - 262: {CoreID: 35, SocketID: 1}, - 89: {CoreID: 42, SocketID: 1}, - 209: {CoreID: 42, SocketID: 1}, - 149: {CoreID: 42, SocketID: 1}, - 269: {CoreID: 42, SocketID: 1}, - 90: {CoreID: 43, SocketID: 1}, - 210: {CoreID: 43, SocketID: 1}, - 150: {CoreID: 43, SocketID: 1}, - 270: {CoreID: 43, SocketID: 1}, - 97: {CoreID: 50, SocketID: 1}, - 217: {CoreID: 50, SocketID: 1}, - 157: {CoreID: 50, SocketID: 1}, - 277: {CoreID: 50, SocketID: 1}, - 98: {CoreID: 51, SocketID: 1}, - 218: {CoreID: 51, SocketID: 1}, - 158: {CoreID: 51, SocketID: 1}, - 278: {CoreID: 51, SocketID: 1}, - 5: {CoreID: 68, SocketID: 2}, - 29: {CoreID: 68, SocketID: 2}, - 17: {CoreID: 68, SocketID: 2}, - 42: {CoreID: 68, SocketID: 2}, - 6: {CoreID: 69, SocketID: 2}, - 30: {CoreID: 69, SocketID: 2}, - 18: {CoreID: 69, SocketID: 2}, - 43: {CoreID: 69, SocketID: 2}, - 53: {CoreID: 4, SocketID: 2}, - 173: {CoreID: 4, SocketID: 2}, - 113: {CoreID: 4, SocketID: 2}, - 233: {CoreID: 4, SocketID: 2}, - 54: {CoreID: 5, SocketID: 2}, - 174: {CoreID: 5, SocketID: 2}, - 114: {CoreID: 5, SocketID: 2}, - 234: {CoreID: 5, SocketID: 2}, - 61: {CoreID: 12, SocketID: 2}, - 181: {CoreID: 12, SocketID: 2}, - 121: {CoreID: 12, SocketID: 2}, - 241: {CoreID: 12, SocketID: 2}, - 62: {CoreID: 13, SocketID: 2}, - 182: {CoreID: 13, SocketID: 2}, - 122: {CoreID: 13, SocketID: 2}, - 242: {CoreID: 13, SocketID: 2}, - 67: {CoreID: 20, SocketID: 2}, - 187: {CoreID: 20, SocketID: 2}, - 127: {CoreID: 20, SocketID: 2}, - 247: {CoreID: 20, SocketID: 2}, - 68: {CoreID: 21, SocketID: 2}, - 188: {CoreID: 21, SocketID: 2}, - 128: {CoreID: 21, SocketID: 2}, - 248: {CoreID: 21, SocketID: 2}, - 75: {CoreID: 28, SocketID: 2}, - 195: {CoreID: 28, SocketID: 2}, - 135: {CoreID: 28, SocketID: 2}, - 255: {CoreID: 28, SocketID: 2}, - 76: {CoreID: 29, SocketID: 2}, - 196: {CoreID: 29, SocketID: 2}, - 136: {CoreID: 29, SocketID: 2}, - 256: {CoreID: 29, SocketID: 2}, - 83: {CoreID: 36, SocketID: 2}, - 203: {CoreID: 36, SocketID: 2}, - 143: {CoreID: 36, SocketID: 2}, - 263: {CoreID: 36, SocketID: 2}, - 84: {CoreID: 37, SocketID: 2}, - 204: {CoreID: 37, SocketID: 2}, - 144: {CoreID: 37, SocketID: 2}, - 264: {CoreID: 37, SocketID: 2}, - 91: {CoreID: 44, SocketID: 2}, - 211: {CoreID: 44, SocketID: 2}, - 151: {CoreID: 44, SocketID: 2}, - 271: {CoreID: 44, SocketID: 2}, - 92: {CoreID: 45, SocketID: 2}, - 212: {CoreID: 45, SocketID: 2}, - 152: {CoreID: 45, SocketID: 2}, - 272: {CoreID: 45, SocketID: 2}, - 99: {CoreID: 52, SocketID: 2}, - 219: {CoreID: 52, SocketID: 2}, - 159: {CoreID: 52, SocketID: 2}, - 279: {CoreID: 52, SocketID: 2}, - 100: {CoreID: 53, SocketID: 2}, - 220: {CoreID: 53, SocketID: 2}, - 160: {CoreID: 53, SocketID: 2}, - 280: {CoreID: 53, SocketID: 2}, - 105: {CoreID: 60, SocketID: 2}, - 225: {CoreID: 60, SocketID: 2}, - 165: {CoreID: 60, SocketID: 2}, - 285: {CoreID: 60, SocketID: 2}, - 106: {CoreID: 61, SocketID: 2}, - 226: {CoreID: 61, SocketID: 2}, - 166: {CoreID: 61, SocketID: 2}, - 286: {CoreID: 61, SocketID: 2}, - 7: {CoreID: 70, SocketID: 3}, - 31: {CoreID: 70, SocketID: 3}, - 19: {CoreID: 70, SocketID: 3}, - 44: {CoreID: 70, SocketID: 3}, - 8: {CoreID: 71, SocketID: 3}, - 32: {CoreID: 71, SocketID: 3}, - 20: {CoreID: 71, SocketID: 3}, - 45: {CoreID: 71, SocketID: 3}, - 37: {CoreID: 63, SocketID: 3}, - 168: {CoreID: 63, SocketID: 3}, - 108: {CoreID: 63, SocketID: 3}, - 228: {CoreID: 63, SocketID: 3}, - 107: {CoreID: 62, SocketID: 3}, - 227: {CoreID: 62, SocketID: 3}, - 167: {CoreID: 62, SocketID: 3}, - 287: {CoreID: 62, SocketID: 3}, - 55: {CoreID: 6, SocketID: 3}, - 175: {CoreID: 6, SocketID: 3}, - 115: {CoreID: 6, SocketID: 3}, - 235: {CoreID: 6, SocketID: 3}, - 56: {CoreID: 7, SocketID: 3}, - 176: {CoreID: 7, SocketID: 3}, - 116: {CoreID: 7, SocketID: 3}, - 236: {CoreID: 7, SocketID: 3}, - 63: {CoreID: 14, SocketID: 3}, - 183: {CoreID: 14, SocketID: 3}, - 123: {CoreID: 14, SocketID: 3}, - 243: {CoreID: 14, SocketID: 3}, - 64: {CoreID: 15, SocketID: 3}, - 184: {CoreID: 15, SocketID: 3}, - 124: {CoreID: 15, SocketID: 3}, - 244: {CoreID: 15, SocketID: 3}, - 69: {CoreID: 22, SocketID: 3}, - 189: {CoreID: 22, SocketID: 3}, - 129: {CoreID: 22, SocketID: 3}, - 249: {CoreID: 22, SocketID: 3}, - 70: {CoreID: 23, SocketID: 3}, - 190: {CoreID: 23, SocketID: 3}, - 130: {CoreID: 23, SocketID: 3}, - 250: {CoreID: 23, SocketID: 3}, - 77: {CoreID: 30, SocketID: 3}, - 197: {CoreID: 30, SocketID: 3}, - 137: {CoreID: 30, SocketID: 3}, - 257: {CoreID: 30, SocketID: 3}, - 78: {CoreID: 31, SocketID: 3}, - 198: {CoreID: 31, SocketID: 3}, - 138: {CoreID: 31, SocketID: 3}, - 258: {CoreID: 31, SocketID: 3}, - 85: {CoreID: 38, SocketID: 3}, - 205: {CoreID: 38, SocketID: 3}, - 145: {CoreID: 38, SocketID: 3}, - 265: {CoreID: 38, SocketID: 3}, - 86: {CoreID: 39, SocketID: 3}, - 206: {CoreID: 39, SocketID: 3}, - 146: {CoreID: 39, SocketID: 3}, - 266: {CoreID: 39, SocketID: 3}, - 93: {CoreID: 46, SocketID: 3}, - 213: {CoreID: 46, SocketID: 3}, - 153: {CoreID: 46, SocketID: 3}, - 273: {CoreID: 46, SocketID: 3}, - 94: {CoreID: 47, SocketID: 3}, - 214: {CoreID: 47, SocketID: 3}, - 154: {CoreID: 47, SocketID: 3}, - 274: {CoreID: 47, SocketID: 3}, - 101: {CoreID: 54, SocketID: 3}, - 221: {CoreID: 54, SocketID: 3}, - 161: {CoreID: 54, SocketID: 3}, - 281: {CoreID: 54, SocketID: 3}, - 102: {CoreID: 55, SocketID: 3}, - 222: {CoreID: 55, SocketID: 3}, - 162: {CoreID: 55, SocketID: 3}, - 282: {CoreID: 55, SocketID: 3}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 169: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 109: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 229: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 50: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 170: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 110: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 230: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 25: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 13: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 38: {CoreID: 64, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 26: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 14: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 39: {CoreID: 65, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 33: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 21: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 46: {CoreID: 72, SocketID: 0, NUMANodeID: 0}, + 10: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 34: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 22: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 47: {CoreID: 73, SocketID: 0, NUMANodeID: 0}, + 57: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 177: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 117: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 237: {CoreID: 8, SocketID: 0, NUMANodeID: 0}, + 58: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 178: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 118: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 238: {CoreID: 9, SocketID: 0, NUMANodeID: 0}, + 71: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 191: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 131: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 251: {CoreID: 24, SocketID: 0, NUMANodeID: 0}, + 72: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 192: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 132: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 252: {CoreID: 25, SocketID: 0, NUMANodeID: 0}, + 79: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 199: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 139: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 259: {CoreID: 32, SocketID: 0, NUMANodeID: 0}, + 80: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 200: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 140: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 260: {CoreID: 33, SocketID: 0, NUMANodeID: 0}, + 87: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 207: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 147: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 267: {CoreID: 40, SocketID: 0, NUMANodeID: 0}, + 88: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 208: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 148: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 268: {CoreID: 41, SocketID: 0, NUMANodeID: 0}, + 95: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 215: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 155: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 275: {CoreID: 48, SocketID: 0, NUMANodeID: 0}, + 96: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 216: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 156: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 276: {CoreID: 49, SocketID: 0, NUMANodeID: 0}, + 103: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 223: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 163: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 283: {CoreID: 56, SocketID: 0, NUMANodeID: 0}, + 104: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 224: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 164: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 284: {CoreID: 57, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 27: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 15: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 40: {CoreID: 66, SocketID: 1, NUMANodeID: 1}, + 4: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 28: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 16: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 41: {CoreID: 67, SocketID: 1, NUMANodeID: 1}, + 11: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 35: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 23: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 48: {CoreID: 74, SocketID: 1, NUMANodeID: 1}, + 12: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 36: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 24: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 49: {CoreID: 75, SocketID: 1, NUMANodeID: 1}, + 51: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 171: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 111: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 231: {CoreID: 2, SocketID: 1, NUMANodeID: 1}, + 52: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 172: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 112: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 232: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 59: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 179: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 119: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 239: {CoreID: 10, SocketID: 1, NUMANodeID: 1}, + 60: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 180: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 120: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 240: {CoreID: 11, SocketID: 1, NUMANodeID: 1}, + 65: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 185: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 125: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 245: {CoreID: 18, SocketID: 1, NUMANodeID: 1}, + 66: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 186: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 126: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 246: {CoreID: 19, SocketID: 1, NUMANodeID: 1}, + 73: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 193: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 133: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 253: {CoreID: 26, SocketID: 1, NUMANodeID: 1}, + 74: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 194: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 134: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 254: {CoreID: 27, SocketID: 1, NUMANodeID: 1}, + 81: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 201: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 141: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 261: {CoreID: 34, SocketID: 1, NUMANodeID: 1}, + 82: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 202: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 142: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 262: {CoreID: 35, SocketID: 1, NUMANodeID: 1}, + 89: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 209: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 149: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 269: {CoreID: 42, SocketID: 1, NUMANodeID: 1}, + 90: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 210: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 150: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 270: {CoreID: 43, SocketID: 1, NUMANodeID: 1}, + 97: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 217: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 157: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 277: {CoreID: 50, SocketID: 1, NUMANodeID: 1}, + 98: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 218: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 158: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 278: {CoreID: 51, SocketID: 1, NUMANodeID: 1}, + 5: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 29: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 17: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 42: {CoreID: 68, SocketID: 2, NUMANodeID: 2}, + 6: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 30: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 18: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 43: {CoreID: 69, SocketID: 2, NUMANodeID: 2}, + 53: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 173: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 113: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 233: {CoreID: 4, SocketID: 2, NUMANodeID: 2}, + 54: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 174: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 114: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 234: {CoreID: 5, SocketID: 2, NUMANodeID: 2}, + 61: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 181: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 121: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 241: {CoreID: 12, SocketID: 2, NUMANodeID: 2}, + 62: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 182: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 122: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 242: {CoreID: 13, SocketID: 2, NUMANodeID: 2}, + 67: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 187: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 127: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 247: {CoreID: 20, SocketID: 2, NUMANodeID: 2}, + 68: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 188: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 128: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 248: {CoreID: 21, SocketID: 2, NUMANodeID: 2}, + 75: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 195: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 135: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 255: {CoreID: 28, SocketID: 2, NUMANodeID: 2}, + 76: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 196: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 136: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 256: {CoreID: 29, SocketID: 2, NUMANodeID: 2}, + 83: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 203: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 143: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 263: {CoreID: 36, SocketID: 2, NUMANodeID: 2}, + 84: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 204: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 144: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 264: {CoreID: 37, SocketID: 2, NUMANodeID: 2}, + 91: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 211: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 151: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 271: {CoreID: 44, SocketID: 2, NUMANodeID: 2}, + 92: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 212: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 152: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 272: {CoreID: 45, SocketID: 2, NUMANodeID: 2}, + 99: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 219: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 159: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 279: {CoreID: 52, SocketID: 2, NUMANodeID: 2}, + 100: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 220: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 160: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 280: {CoreID: 53, SocketID: 2, NUMANodeID: 2}, + 105: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 225: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 165: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 285: {CoreID: 60, SocketID: 2, NUMANodeID: 2}, + 106: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 226: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 166: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 286: {CoreID: 61, SocketID: 2, NUMANodeID: 2}, + 7: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 31: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 19: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 44: {CoreID: 70, SocketID: 3, NUMANodeID: 3}, + 8: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 32: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 20: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 45: {CoreID: 71, SocketID: 3, NUMANodeID: 3}, + 37: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 168: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 108: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 228: {CoreID: 63, SocketID: 3, NUMANodeID: 3}, + 107: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 227: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 167: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 287: {CoreID: 62, SocketID: 3, NUMANodeID: 3}, + 55: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 175: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 115: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 235: {CoreID: 6, SocketID: 3, NUMANodeID: 3}, + 56: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 176: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 116: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 236: {CoreID: 7, SocketID: 3, NUMANodeID: 3}, + 63: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 183: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 123: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 243: {CoreID: 14, SocketID: 3, NUMANodeID: 3}, + 64: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 184: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 124: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 244: {CoreID: 15, SocketID: 3, NUMANodeID: 3}, + 69: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 189: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 129: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 249: {CoreID: 22, SocketID: 3, NUMANodeID: 3}, + 70: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 190: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 130: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 250: {CoreID: 23, SocketID: 3, NUMANodeID: 3}, + 77: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 197: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 137: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 257: {CoreID: 30, SocketID: 3, NUMANodeID: 3}, + 78: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 198: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 138: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 258: {CoreID: 31, SocketID: 3, NUMANodeID: 3}, + 85: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 205: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 145: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 265: {CoreID: 38, SocketID: 3, NUMANodeID: 3}, + 86: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 206: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 146: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 266: {CoreID: 39, SocketID: 3, NUMANodeID: 3}, + 93: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 213: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 153: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 273: {CoreID: 46, SocketID: 3, NUMANodeID: 3}, + 94: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 214: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 154: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 274: {CoreID: 47, SocketID: 3, NUMANodeID: 3}, + 101: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 221: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 161: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 281: {CoreID: 54, SocketID: 3, NUMANodeID: 3}, + 102: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, + 222: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, + 162: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, + 282: {CoreID: 55, SocketID: 3, NUMANodeID: 3}, }, } ) diff --git a/pkg/kubelet/cm/cpumanager/topology/BUILD b/pkg/kubelet/cm/cpumanager/topology/BUILD index 18ab41a33f4..bd5fb095200 100644 --- a/pkg/kubelet/cm/cpumanager/topology/BUILD +++ b/pkg/kubelet/cm/cpumanager/topology/BUILD @@ -33,5 +33,8 @@ go_test( name = "go_default_test", srcs = ["topology_test.go"], embed = [":go_default_library"], - deps = ["//vendor/github.com/google/cadvisor/info/v1:go_default_library"], + deps = [ + "//pkg/kubelet/cm/cpuset:go_default_library", + "//vendor/github.com/google/cadvisor/info/v1:go_default_library", + ], ) diff --git a/pkg/kubelet/cm/cpumanager/topology/topology.go b/pkg/kubelet/cm/cpumanager/topology/topology.go index f19acac5fcb..abde76931bf 100644 --- a/pkg/kubelet/cm/cpumanager/topology/topology.go +++ b/pkg/kubelet/cm/cpumanager/topology/topology.go @@ -18,13 +18,19 @@ package topology import ( "fmt" + "io/ioutil" + "strings" cadvisorapi "github.com/google/cadvisor/info/v1" "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) -// CPUDetails is a map from CPU ID to Core ID and Socket ID. +// NUMANodeInfo is a map from NUMANode ID to a list of CPU IDs associated with +// that NUMANode. +type NUMANodeInfo map[int]cpuset.CPUSet + +// CPUDetails is a map from CPU ID to Core ID, Socket ID, and NUMA ID. type CPUDetails map[int]CPUInfo // CPUTopology contains details of node cpu, where : @@ -56,10 +62,11 @@ func (topo *CPUTopology) CPUsPerSocket() int { return topo.NumCPUs / topo.NumSockets } -// CPUInfo contains the socket and core IDs associated with a CPU. +// CPUInfo contains the NUMA, socket, and core IDs associated with a CPU. type CPUInfo struct { - SocketID int - CoreID int + NUMANodeID int + SocketID int + CoreID int } // KeepOnly returns a new CPUDetails object with only the supplied cpus. @@ -73,6 +80,28 @@ func (d CPUDetails) KeepOnly(cpus cpuset.CPUSet) CPUDetails { return result } +// NUMANodes returns all of the NUMANode IDs associated with the CPUs in this +// CPUDetails. +func (d CPUDetails) NUMANodes() cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + b.Add(info.NUMANodeID) + } + return b.Result() +} + +// NUMANodesInSocket returns all of the logical NUMANode IDs associated with +// the given Socket ID in this CPUDetails. +func (d CPUDetails) NUMANodesInSocket(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + if info.SocketID == id { + b.Add(info.NUMANodeID) + } + } + return b.Result() +} + // Sockets returns all of the socket IDs associated with the CPUs in this // CPUDetails. func (d CPUDetails) Sockets() cpuset.CPUSet { @@ -95,6 +124,18 @@ func (d CPUDetails) CPUsInSocket(id int) cpuset.CPUSet { return b.Result() } +// SocketsInNUMANode returns all of the logical Socket IDs associated with the +// given NUMANode ID in this CPUDetails. +func (d CPUDetails) SocketsInNUMANode(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + if info.NUMANodeID == id { + b.Add(info.SocketID) + } + } + return b.Result() +} + // Cores returns all of the core IDs associated with the CPUs in this // CPUDetails. func (d CPUDetails) Cores() cpuset.CPUSet { @@ -105,6 +146,18 @@ func (d CPUDetails) Cores() cpuset.CPUSet { return b.Result() } +// CoresInNUMANode returns all of the core IDs associated with the given +// NUMA ID in this CPUDetails. +func (d CPUDetails) CoresInNUMANode(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for _, info := range d { + if info.NUMANodeID == id { + b.Add(info.CoreID) + } + } + return b.Result() +} + // CoresInSocket returns all of the core IDs associated with the given // socket ID in this CPUDetails. func (d CPUDetails) CoresInSocket(id int) cpuset.CPUSet { @@ -126,6 +179,18 @@ func (d CPUDetails) CPUs() cpuset.CPUSet { return b.Result() } +// CPUsInNUMANode returns all of the logical CPU IDs associated with the given +// NUMANode ID in this CPUDetails. +func (d CPUDetails) CPUsInNUMANode(id int) cpuset.CPUSet { + b := cpuset.NewBuilder() + for cpu, info := range d { + if info.NUMANodeID == id { + b.Add(cpu) + } + } + return b.Result() +} + // CPUsInCore returns all of the logical CPU IDs associated with the // given core ID in this CPUDetails. func (d CPUDetails) CPUsInCore(id int) cpuset.CPUSet { @@ -139,7 +204,7 @@ func (d CPUDetails) CPUsInCore(id int) cpuset.CPUSet { } // Discover returns CPUTopology based on cadvisor node info -func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) { +func Discover(machineInfo *cadvisorapi.MachineInfo, numaNodeInfo NUMANodeInfo) (*CPUTopology, error) { if machineInfo.NumCores == 0 { return nil, fmt.Errorf("could not detect number of cpus") } @@ -152,9 +217,16 @@ func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) { for _, core := range socket.Cores { if coreID, err := getUniqueCoreID(core.Threads); err == nil { for _, cpu := range core.Threads { + numaNodeID := 0 + for id, cset := range numaNodeInfo { + if cset.Contains(cpu) { + numaNodeID = id + } + } CPUDetails[cpu] = CPUInfo{ - CoreID: coreID, - SocketID: socket.Id, + CoreID: coreID, + SocketID: socket.Id, + NUMANodeID: numaNodeID, } } } else { @@ -194,3 +266,49 @@ func getUniqueCoreID(threads []int) (coreID int, err error) { return min, nil } + +// GetNUMANodeInfo uses sysfs to return a map of NUMANode id to the list of +// CPUs associated with that NUMANode. +// +// TODO: This is a temporary workaround until cadvisor provides this +// information directly in machineInfo. We should remove this once this +// information is available from cadvisor. +func GetNUMANodeInfo() (NUMANodeInfo, error) { + // Get the possible NUMA nodes on this machine. If reading this file + // is not possible, this is not an error. Instead, we just return a + // nil NUMANodeInfo, indicating that no NUMA information is available + // on this machine. This should implicitly be interpreted as having a + // single NUMA node with id 0 for all CPUs. + nodelist, err := ioutil.ReadFile("/sys/devices/system/node/possible") + if err != nil { + return nil, nil + } + + // Parse the nodelist into a set of Node IDs + nodes, err := cpuset.Parse(strings.TrimSpace(string(nodelist))) + if err != nil { + return nil, err + } + + info := make(NUMANodeInfo) + + // For each node... + for _, node := range nodes.ToSlice() { + // Read the 'cpulist' of the NUMA node from sysfs. + path := fmt.Sprintf("/sys/devices/system/node/node%d/cpulist", node) + cpulist, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + // Convert the 'cpulist' into a set of CPUs. + cpus, err := cpuset.Parse(strings.TrimSpace(string(cpulist))) + if err != nil { + return nil, err + } + + info[node] = cpus + } + + return info, nil +} diff --git a/pkg/kubelet/cm/cpumanager/topology/topology_test.go b/pkg/kubelet/cm/cpumanager/topology/topology_test.go index c8441ed245d..59be5e02861 100644 --- a/pkg/kubelet/cm/cpumanager/topology/topology_test.go +++ b/pkg/kubelet/cm/cpumanager/topology/topology_test.go @@ -21,27 +21,30 @@ import ( "testing" cadvisorapi "github.com/google/cadvisor/info/v1" + "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) func Test_Discover(t *testing.T) { tests := []struct { - name string - args *cadvisorapi.MachineInfo - want *CPUTopology - wantErr bool + name string + machineInfo cadvisorapi.MachineInfo + numaNodeInfo NUMANodeInfo + want *CPUTopology + wantErr bool }{ { name: "FailNumCores", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 0, }, - want: &CPUTopology{}, - wantErr: true, + numaNodeInfo: NUMANodeInfo{}, + want: &CPUTopology{}, + wantErr: true, }, { name: "OneSocketHT", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 8, Topology: []cadvisorapi.Node{ {Id: 0, @@ -54,26 +57,29 @@ func Test_Discover(t *testing.T) { }, }, }, + numaNodeInfo: NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7), + }, want: &CPUTopology{ NumCPUs: 8, NumSockets: 1, NumCores: 4, CPUDetails: map[int]CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 0}, - 4: {CoreID: 0, SocketID: 0}, - 5: {CoreID: 1, SocketID: 0}, - 6: {CoreID: 2, SocketID: 0}, - 7: {CoreID: 3, SocketID: 0}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, + 4: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 5: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 6: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 3, SocketID: 0, NUMANodeID: 0}, }, }, wantErr: false, }, { name: "DualSocketNoHT", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 4, Topology: []cadvisorapi.Node{ {Id: 0, @@ -90,22 +96,26 @@ func Test_Discover(t *testing.T) { }, }, }, + numaNodeInfo: NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 2), + 1: cpuset.NewCPUSet(1, 3), + }, want: &CPUTopology{ NumCPUs: 4, NumSockets: 2, NumCores: 4, CPUDetails: map[int]CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 1}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 1, NUMANodeID: 1}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, }, }, wantErr: false, }, { name: "DualSocketHT - non unique Core'ID's", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 12, Topology: []cadvisorapi.Node{ {Id: 0, @@ -124,30 +134,34 @@ func Test_Discover(t *testing.T) { }, }, }, + numaNodeInfo: NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8), + 1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11), + }, want: &CPUTopology{ NumCPUs: 12, NumSockets: 2, NumCores: 6, CPUDetails: map[int]CPUInfo{ - 0: {CoreID: 0, SocketID: 0}, - 1: {CoreID: 1, SocketID: 0}, - 2: {CoreID: 2, SocketID: 0}, - 3: {CoreID: 3, SocketID: 1}, - 4: {CoreID: 4, SocketID: 1}, - 5: {CoreID: 5, SocketID: 1}, - 6: {CoreID: 0, SocketID: 0}, - 7: {CoreID: 1, SocketID: 0}, - 8: {CoreID: 2, SocketID: 0}, - 9: {CoreID: 3, SocketID: 1}, - 10: {CoreID: 4, SocketID: 1}, - 11: {CoreID: 5, SocketID: 1}, + 0: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 1: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 2: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 3: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 4: {CoreID: 4, SocketID: 1, NUMANodeID: 1}, + 5: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, + 6: {CoreID: 0, SocketID: 0, NUMANodeID: 0}, + 7: {CoreID: 1, SocketID: 0, NUMANodeID: 0}, + 8: {CoreID: 2, SocketID: 0, NUMANodeID: 0}, + 9: {CoreID: 3, SocketID: 1, NUMANodeID: 1}, + 10: {CoreID: 4, SocketID: 1, NUMANodeID: 1}, + 11: {CoreID: 5, SocketID: 1, NUMANodeID: 1}, }, }, wantErr: false, }, { name: "OneSocketHT fail", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 8, Topology: []cadvisorapi.Node{ {Id: 0, @@ -160,12 +174,13 @@ func Test_Discover(t *testing.T) { }, }, }, - want: &CPUTopology{}, - wantErr: true, + numaNodeInfo: NUMANodeInfo{}, + want: &CPUTopology{}, + wantErr: true, }, { name: "OneSocketHT fail", - args: &cadvisorapi.MachineInfo{ + machineInfo: cadvisorapi.MachineInfo{ NumCores: 8, Topology: []cadvisorapi.Node{ {Id: 0, @@ -178,13 +193,14 @@ func Test_Discover(t *testing.T) { }, }, }, - want: &CPUTopology{}, - wantErr: true, + numaNodeInfo: NUMANodeInfo{}, + want: &CPUTopology{}, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := Discover(tt.args) + got, err := Discover(&tt.machineInfo, tt.numaNodeInfo) if err != nil { if tt.wantErr { t.Logf("Discover() expected error = %v", err) diff --git a/pkg/kubelet/cm/cpumanager/topology_hints_test.go b/pkg/kubelet/cm/cpumanager/topology_hints_test.go index 8b340f3f007..8b639dd6000 100644 --- a/pkg/kubelet/cm/cpumanager/topology_hints_test.go +++ b/pkg/kubelet/cm/cpumanager/topology_hints_test.go @@ -70,7 +70,12 @@ func TestGetTopologyHints(t *testing.T) { }, } - topology, _ := topology.Discover(&machineInfo) + numaNodeInfo := topology.NUMANodeInfo{ + 0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8), + 1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11), + } + + topology, _ := topology.Discover(&machineInfo, numaNodeInfo) m := manager{ policy: &staticPolicy{