mirror of
https://github.com/k3s-io/kubernetes.git
synced 2025-07-28 14:07:14 +00:00
Merge pull request #90980 from AlexeyPerevalov/GetNUMANodeInfo
Avoid using socket for hints in generateCPUTopologyHints
This commit is contained in:
commit
81bf1f8789
@ -56,7 +56,6 @@ go_library(
|
|||||||
"//pkg/apis/core/v1/helper/qos:go_default_library",
|
"//pkg/apis/core/v1/helper/qos:go_default_library",
|
||||||
"//pkg/kubelet/cadvisor:go_default_library",
|
"//pkg/kubelet/cadvisor:go_default_library",
|
||||||
"//pkg/kubelet/cm/containermap:go_default_library",
|
"//pkg/kubelet/cm/containermap:go_default_library",
|
||||||
"//pkg/kubelet/cm/cpumanager/topology:go_default_library",
|
|
||||||
"//pkg/kubelet/cm/devicemanager:go_default_library",
|
"//pkg/kubelet/cm/devicemanager:go_default_library",
|
||||||
"//pkg/kubelet/cm/util:go_default_library",
|
"//pkg/kubelet/cm/util:go_default_library",
|
||||||
"//pkg/kubelet/events:go_default_library",
|
"//pkg/kubelet/events:go_default_library",
|
||||||
@ -118,7 +117,6 @@ go_library(
|
|||||||
"//pkg/apis/core/v1/helper/qos:go_default_library",
|
"//pkg/apis/core/v1/helper/qos:go_default_library",
|
||||||
"//pkg/kubelet/cadvisor:go_default_library",
|
"//pkg/kubelet/cadvisor:go_default_library",
|
||||||
"//pkg/kubelet/cm/containermap:go_default_library",
|
"//pkg/kubelet/cm/containermap:go_default_library",
|
||||||
"//pkg/kubelet/cm/cpumanager/topology:go_default_library",
|
|
||||||
"//pkg/kubelet/cm/devicemanager:go_default_library",
|
"//pkg/kubelet/cm/devicemanager:go_default_library",
|
||||||
"//pkg/kubelet/cm/util:go_default_library",
|
"//pkg/kubelet/cm/util:go_default_library",
|
||||||
"//pkg/kubelet/events:go_default_library",
|
"//pkg/kubelet/events:go_default_library",
|
||||||
|
@ -52,7 +52,6 @@ import (
|
|||||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/containermap"
|
"k8s.io/kubernetes/pkg/kubelet/cm/containermap"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
|
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
|
||||||
cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
|
"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
||||||
cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util"
|
cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util"
|
||||||
@ -238,13 +237,6 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Correct NUMA information is currently missing from cadvisor's
|
|
||||||
// MachineInfo struct, so we use the CPUManager's internal logic for
|
|
||||||
// gathering NUMANodeInfo to pass to components that care about it.
|
|
||||||
numaNodeInfo, err := cputopology.GetNUMANodeInfo()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
capacity := cadvisor.CapacityFromMachineInfo(machineInfo)
|
capacity := cadvisor.CapacityFromMachineInfo(machineInfo)
|
||||||
for k, v := range capacity {
|
for k, v := range capacity {
|
||||||
internalCapacity[k] = v
|
internalCapacity[k] = v
|
||||||
@ -300,7 +292,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
|
|||||||
|
|
||||||
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.TopologyManager) {
|
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.TopologyManager) {
|
||||||
cm.topologyManager, err = topologymanager.NewManager(
|
cm.topologyManager, err = topologymanager.NewManager(
|
||||||
numaNodeInfo,
|
machineInfo.Topology,
|
||||||
nodeConfig.ExperimentalTopologyManagerPolicy,
|
nodeConfig.ExperimentalTopologyManagerPolicy,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -315,7 +307,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
|
|||||||
|
|
||||||
klog.Infof("Creating device plugin manager: %t", devicePluginEnabled)
|
klog.Infof("Creating device plugin manager: %t", devicePluginEnabled)
|
||||||
if devicePluginEnabled {
|
if devicePluginEnabled {
|
||||||
cm.deviceManager, err = devicemanager.NewManagerImpl(numaNodeInfo, cm.topologyManager)
|
cm.deviceManager, err = devicemanager.NewManagerImpl(machineInfo.Topology, cm.topologyManager)
|
||||||
cm.topologyManager.AddHintProvider(cm.deviceManager)
|
cm.topologyManager.AddHintProvider(cm.deviceManager)
|
||||||
} else {
|
} else {
|
||||||
cm.deviceManager, err = devicemanager.NewManagerStub()
|
cm.deviceManager, err = devicemanager.NewManagerStub()
|
||||||
@ -330,7 +322,6 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
|
|||||||
nodeConfig.ExperimentalCPUManagerPolicy,
|
nodeConfig.ExperimentalCPUManagerPolicy,
|
||||||
nodeConfig.ExperimentalCPUManagerReconcilePeriod,
|
nodeConfig.ExperimentalCPUManagerReconcilePeriod,
|
||||||
machineInfo,
|
machineInfo,
|
||||||
numaNodeInfo,
|
|
||||||
nodeConfig.NodeAllocatableConfig.ReservedSystemCPUs,
|
nodeConfig.NodeAllocatableConfig.ReservedSystemCPUs,
|
||||||
cm.GetNodeAllocatableReservation(),
|
cm.GetNodeAllocatableReservation(),
|
||||||
nodeConfig.KubeletRootDir,
|
nodeConfig.KubeletRootDir,
|
||||||
|
@ -126,7 +126,7 @@ func (s *sourcesReadyStub) AddSource(source string) {}
|
|||||||
func (s *sourcesReadyStub) AllReady() bool { return true }
|
func (s *sourcesReadyStub) AllReady() bool { return true }
|
||||||
|
|
||||||
// NewManager creates new cpu manager based on provided policy
|
// NewManager creates new cpu manager based on provided policy
|
||||||
func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, numaNodeInfo topology.NUMANodeInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
|
func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, specificCPUs cpuset.CPUSet, nodeAllocatableReservation v1.ResourceList, stateFileDirectory string, affinity topologymanager.Store) (Manager, error) {
|
||||||
var topo *topology.CPUTopology
|
var topo *topology.CPUTopology
|
||||||
var policy Policy
|
var policy Policy
|
||||||
|
|
||||||
@ -137,7 +137,7 @@ func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo
|
|||||||
|
|
||||||
case PolicyStatic:
|
case PolicyStatic:
|
||||||
var err error
|
var err error
|
||||||
topo, err = topology.Discover(machineInfo, numaNodeInfo)
|
topo, err = topology.Discover(machineInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -629,7 +629,7 @@ func TestCPUManagerGenerate(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer os.RemoveAll(sDir)
|
defer os.RemoveAll(sDir)
|
||||||
|
|
||||||
mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, nil, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
|
mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, cpuset.NewCPUSet(), testCase.nodeAllocatableReservation, sDir, topologymanager.NewFakeManager())
|
||||||
if testCase.expectedError != nil {
|
if testCase.expectedError != nil {
|
||||||
if !strings.Contains(err.Error(), testCase.expectedError.Error()) {
|
if !strings.Contains(err.Error(), testCase.expectedError.Error()) {
|
||||||
t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error())
|
t.Errorf("Unexpected error message. Have: %s wants %s", err.Error(), testCase.expectedError.Error())
|
||||||
|
@ -364,24 +364,18 @@ func (p *staticPolicy) GetTopologyHints(s state.State, pod *v1.Pod, container *v
|
|||||||
func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reusableCPUs cpuset.CPUSet, request int) []topologymanager.TopologyHint {
|
func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reusableCPUs cpuset.CPUSet, request int) []topologymanager.TopologyHint {
|
||||||
// Initialize minAffinitySize to include all NUMA Nodes.
|
// Initialize minAffinitySize to include all NUMA Nodes.
|
||||||
minAffinitySize := p.topology.CPUDetails.NUMANodes().Size()
|
minAffinitySize := p.topology.CPUDetails.NUMANodes().Size()
|
||||||
// Initialize minSocketsOnMinAffinity to include all Sockets.
|
|
||||||
minSocketsOnMinAffinity := p.topology.CPUDetails.Sockets().Size()
|
|
||||||
|
|
||||||
// Iterate through all combinations of socket bitmask and build hints from them.
|
// Iterate through all combinations of numa nodes bitmask and build hints from them.
|
||||||
hints := []topologymanager.TopologyHint{}
|
hints := []topologymanager.TopologyHint{}
|
||||||
bitmask.IterateBitMasks(p.topology.CPUDetails.NUMANodes().ToSlice(), func(mask bitmask.BitMask) {
|
bitmask.IterateBitMasks(p.topology.CPUDetails.NUMANodes().ToSlice(), func(mask bitmask.BitMask) {
|
||||||
// First, update minAffinitySize and minSocketsOnMinAffinity for the
|
// First, update minAffinitySize for the current request size.
|
||||||
// current request size.
|
|
||||||
cpusInMask := p.topology.CPUDetails.CPUsInNUMANodes(mask.GetBits()...).Size()
|
cpusInMask := p.topology.CPUDetails.CPUsInNUMANodes(mask.GetBits()...).Size()
|
||||||
socketsInMask := p.topology.CPUDetails.SocketsInNUMANodes(mask.GetBits()...).Size()
|
|
||||||
if cpusInMask >= request && mask.Count() < minAffinitySize {
|
if cpusInMask >= request && mask.Count() < minAffinitySize {
|
||||||
minAffinitySize = mask.Count()
|
minAffinitySize = mask.Count()
|
||||||
if socketsInMask < minSocketsOnMinAffinity {
|
|
||||||
minSocketsOnMinAffinity = socketsInMask
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Then check to see if all of the reusable CPUs are part of the bitmask.
|
// Then check to see if we have enough CPUs available on the current
|
||||||
|
// numa node bitmask to satisfy the CPU request.
|
||||||
numMatching := 0
|
numMatching := 0
|
||||||
for _, c := range reusableCPUs.ToSlice() {
|
for _, c := range reusableCPUs.ToSlice() {
|
||||||
// Disregard this mask if its NUMANode isn't part of it.
|
// Disregard this mask if its NUMANode isn't part of it.
|
||||||
@ -404,7 +398,7 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Otherwise, create a new hint from the socket bitmask and add it to the
|
// Otherwise, create a new hint from the numa node bitmask and add it to the
|
||||||
// list of hints. We set all hint preferences to 'false' on the first
|
// list of hints. We set all hint preferences to 'false' on the first
|
||||||
// pass through.
|
// pass through.
|
||||||
hints = append(hints, topologymanager.TopologyHint{
|
hints = append(hints, topologymanager.TopologyHint{
|
||||||
@ -416,14 +410,10 @@ func (p *staticPolicy) generateCPUTopologyHints(availableCPUs cpuset.CPUSet, reu
|
|||||||
// Loop back through all hints and update the 'Preferred' field based on
|
// Loop back through all hints and update the 'Preferred' field based on
|
||||||
// counting the number of bits sets in the affinity mask and comparing it
|
// counting the number of bits sets in the affinity mask and comparing it
|
||||||
// to the minAffinitySize. Only those with an equal number of bits set (and
|
// to the minAffinitySize. Only those with an equal number of bits set (and
|
||||||
// with a minimal set of sockets) will be considered preferred.
|
// with a minimal set of numa nodes) will be considered preferred.
|
||||||
for i := range hints {
|
for i := range hints {
|
||||||
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
|
if hints[i].NUMANodeAffinity.Count() == minAffinitySize {
|
||||||
nodes := hints[i].NUMANodeAffinity.GetBits()
|
hints[i].Preferred = true
|
||||||
numSockets := p.topology.CPUDetails.SocketsInNUMANodes(nodes...).Size()
|
|
||||||
if numSockets == minSocketsOnMinAffinity {
|
|
||||||
hints[i].Preferred = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -33,8 +33,5 @@ go_test(
|
|||||||
name = "go_default_test",
|
name = "go_default_test",
|
||||||
srcs = ["topology_test.go"],
|
srcs = ["topology_test.go"],
|
||||||
embed = [":go_default_library"],
|
embed = [":go_default_library"],
|
||||||
deps = [
|
deps = ["//vendor/github.com/google/cadvisor/info/v1:go_default_library"],
|
||||||
"//pkg/kubelet/cm/cpuset:go_default_library",
|
|
||||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
|
@ -18,8 +18,6 @@ package topology
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
cadvisorapi "github.com/google/cadvisor/info/v1"
|
cadvisorapi "github.com/google/cadvisor/info/v1"
|
||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
@ -218,7 +216,7 @@ func (d CPUDetails) CPUsInCores(ids ...int) cpuset.CPUSet {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Discover returns CPUTopology based on cadvisor node info
|
// Discover returns CPUTopology based on cadvisor node info
|
||||||
func Discover(machineInfo *cadvisorapi.MachineInfo, numaNodeInfo NUMANodeInfo) (*CPUTopology, error) {
|
func Discover(machineInfo *cadvisorapi.MachineInfo) (*CPUTopology, error) {
|
||||||
if machineInfo.NumCores == 0 {
|
if machineInfo.NumCores == 0 {
|
||||||
return nil, fmt.Errorf("could not detect number of cpus")
|
return nil, fmt.Errorf("could not detect number of cpus")
|
||||||
}
|
}
|
||||||
@ -226,26 +224,20 @@ func Discover(machineInfo *cadvisorapi.MachineInfo, numaNodeInfo NUMANodeInfo) (
|
|||||||
CPUDetails := CPUDetails{}
|
CPUDetails := CPUDetails{}
|
||||||
numPhysicalCores := 0
|
numPhysicalCores := 0
|
||||||
|
|
||||||
for _, socket := range machineInfo.Topology {
|
for _, node := range machineInfo.Topology {
|
||||||
numPhysicalCores += len(socket.Cores)
|
numPhysicalCores += len(node.Cores)
|
||||||
for _, core := range socket.Cores {
|
for _, core := range node.Cores {
|
||||||
if coreID, err := getUniqueCoreID(core.Threads); err == nil {
|
if coreID, err := getUniqueCoreID(core.Threads); err == nil {
|
||||||
for _, cpu := range core.Threads {
|
for _, cpu := range core.Threads {
|
||||||
numaNodeID := 0
|
|
||||||
for id, cset := range numaNodeInfo {
|
|
||||||
if cset.Contains(cpu) {
|
|
||||||
numaNodeID = id
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CPUDetails[cpu] = CPUInfo{
|
CPUDetails[cpu] = CPUInfo{
|
||||||
CoreID: coreID,
|
CoreID: coreID,
|
||||||
SocketID: socket.Id,
|
SocketID: core.SocketID,
|
||||||
NUMANodeID: numaNodeID,
|
NUMANodeID: node.Id,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
klog.Errorf("could not get unique coreID for socket: %d core %d threads: %v",
|
klog.Errorf("could not get unique coreID for socket: %d core %d threads: %v",
|
||||||
socket.Id, core.Id, core.Threads)
|
core.SocketID, core.Id, core.Threads)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -280,49 +272,3 @@ func getUniqueCoreID(threads []int) (coreID int, err error) {
|
|||||||
|
|
||||||
return min, nil
|
return min, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetNUMANodeInfo uses sysfs to return a map of NUMANode id to the list of
|
|
||||||
// CPUs associated with that NUMANode.
|
|
||||||
//
|
|
||||||
// TODO: This is a temporary workaround until cadvisor provides this
|
|
||||||
// information directly in machineInfo. We should remove this once this
|
|
||||||
// information is available from cadvisor.
|
|
||||||
func GetNUMANodeInfo() (NUMANodeInfo, error) {
|
|
||||||
// Get the possible NUMA nodes on this machine. If reading this file
|
|
||||||
// is not possible, this is not an error. Instead, we just return a
|
|
||||||
// nil NUMANodeInfo, indicating that no NUMA information is available
|
|
||||||
// on this machine. This should implicitly be interpreted as having a
|
|
||||||
// single NUMA node with id 0 for all CPUs.
|
|
||||||
nodelist, err := ioutil.ReadFile("/sys/devices/system/node/online")
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the nodelist into a set of Node IDs
|
|
||||||
nodes, err := cpuset.Parse(strings.TrimSpace(string(nodelist)))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info := make(NUMANodeInfo)
|
|
||||||
|
|
||||||
// For each node...
|
|
||||||
for _, node := range nodes.ToSlice() {
|
|
||||||
// Read the 'cpulist' of the NUMA node from sysfs.
|
|
||||||
path := fmt.Sprintf("/sys/devices/system/node/node%d/cpulist", node)
|
|
||||||
cpulist, err := ioutil.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert the 'cpulist' into a set of CPUs.
|
|
||||||
cpus, err := cpuset.Parse(strings.TrimSpace(string(cpulist)))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info[node] = cpus
|
|
||||||
}
|
|
||||||
|
|
||||||
return info, nil
|
|
||||||
}
|
|
||||||
|
@ -21,26 +21,23 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
cadvisorapi "github.com/google/cadvisor/info/v1"
|
cadvisorapi "github.com/google/cadvisor/info/v1"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func Test_Discover(t *testing.T) {
|
func Test_Discover(t *testing.T) {
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
machineInfo cadvisorapi.MachineInfo
|
machineInfo cadvisorapi.MachineInfo
|
||||||
numaNodeInfo NUMANodeInfo
|
want *CPUTopology
|
||||||
want *CPUTopology
|
wantErr bool
|
||||||
wantErr bool
|
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "FailNumCores",
|
name: "FailNumCores",
|
||||||
machineInfo: cadvisorapi.MachineInfo{
|
machineInfo: cadvisorapi.MachineInfo{
|
||||||
NumCores: 0,
|
NumCores: 0,
|
||||||
},
|
},
|
||||||
numaNodeInfo: NUMANodeInfo{},
|
want: &CPUTopology{},
|
||||||
want: &CPUTopology{},
|
wantErr: true,
|
||||||
wantErr: true,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "OneSocketHT",
|
name: "OneSocketHT",
|
||||||
@ -49,17 +46,14 @@ func Test_Discover(t *testing.T) {
|
|||||||
Topology: []cadvisorapi.Node{
|
Topology: []cadvisorapi.Node{
|
||||||
{Id: 0,
|
{Id: 0,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{0, 4}},
|
{SocketID: 0, Id: 0, Threads: []int{0, 4}},
|
||||||
{Id: 1, Threads: []int{1, 5}},
|
{SocketID: 0, Id: 1, Threads: []int{1, 5}},
|
||||||
{Id: 2, Threads: []int{2, 6}},
|
{SocketID: 0, Id: 2, Threads: []int{2, 6}},
|
||||||
{Id: 3, Threads: []int{3, 7}},
|
{SocketID: 0, Id: 3, Threads: []int{3, 7}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
numaNodeInfo: NUMANodeInfo{
|
|
||||||
0: cpuset.NewCPUSet(0, 1, 2, 3, 4, 5, 6, 7),
|
|
||||||
},
|
|
||||||
want: &CPUTopology{
|
want: &CPUTopology{
|
||||||
NumCPUs: 8,
|
NumCPUs: 8,
|
||||||
NumSockets: 1,
|
NumSockets: 1,
|
||||||
@ -84,22 +78,18 @@ func Test_Discover(t *testing.T) {
|
|||||||
Topology: []cadvisorapi.Node{
|
Topology: []cadvisorapi.Node{
|
||||||
{Id: 0,
|
{Id: 0,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{0}},
|
{SocketID: 0, Id: 0, Threads: []int{0}},
|
||||||
{Id: 2, Threads: []int{2}},
|
{SocketID: 0, Id: 2, Threads: []int{2}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{Id: 1,
|
{Id: 1,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 1, Threads: []int{1}},
|
{SocketID: 1, Id: 1, Threads: []int{1}},
|
||||||
{Id: 3, Threads: []int{3}},
|
{SocketID: 1, Id: 3, Threads: []int{3}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
numaNodeInfo: NUMANodeInfo{
|
|
||||||
0: cpuset.NewCPUSet(0, 2),
|
|
||||||
1: cpuset.NewCPUSet(1, 3),
|
|
||||||
},
|
|
||||||
want: &CPUTopology{
|
want: &CPUTopology{
|
||||||
NumCPUs: 4,
|
NumCPUs: 4,
|
||||||
NumSockets: 2,
|
NumSockets: 2,
|
||||||
@ -120,24 +110,20 @@ func Test_Discover(t *testing.T) {
|
|||||||
Topology: []cadvisorapi.Node{
|
Topology: []cadvisorapi.Node{
|
||||||
{Id: 0,
|
{Id: 0,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{0, 6}},
|
{SocketID: 0, Id: 0, Threads: []int{0, 6}},
|
||||||
{Id: 1, Threads: []int{1, 7}},
|
{SocketID: 0, Id: 1, Threads: []int{1, 7}},
|
||||||
{Id: 2, Threads: []int{2, 8}},
|
{SocketID: 0, Id: 2, Threads: []int{2, 8}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{Id: 1,
|
{Id: 1,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{3, 9}},
|
{SocketID: 1, Id: 0, Threads: []int{3, 9}},
|
||||||
{Id: 1, Threads: []int{4, 10}},
|
{SocketID: 1, Id: 1, Threads: []int{4, 10}},
|
||||||
{Id: 2, Threads: []int{5, 11}},
|
{SocketID: 1, Id: 2, Threads: []int{5, 11}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
numaNodeInfo: NUMANodeInfo{
|
|
||||||
0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8),
|
|
||||||
1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11),
|
|
||||||
},
|
|
||||||
want: &CPUTopology{
|
want: &CPUTopology{
|
||||||
NumCPUs: 12,
|
NumCPUs: 12,
|
||||||
NumSockets: 2,
|
NumSockets: 2,
|
||||||
@ -166,17 +152,16 @@ func Test_Discover(t *testing.T) {
|
|||||||
Topology: []cadvisorapi.Node{
|
Topology: []cadvisorapi.Node{
|
||||||
{Id: 0,
|
{Id: 0,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{0, 4}},
|
{SocketID: 0, Id: 0, Threads: []int{0, 4}},
|
||||||
{Id: 1, Threads: []int{1, 5}},
|
{SocketID: 0, Id: 1, Threads: []int{1, 5}},
|
||||||
{Id: 2, Threads: []int{2, 2}}, // Wrong case - should fail here
|
{SocketID: 0, Id: 2, Threads: []int{2, 2}}, // Wrong case - should fail here
|
||||||
{Id: 3, Threads: []int{3, 7}},
|
{SocketID: 0, Id: 3, Threads: []int{3, 7}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
numaNodeInfo: NUMANodeInfo{},
|
want: &CPUTopology{},
|
||||||
want: &CPUTopology{},
|
wantErr: true,
|
||||||
wantErr: true,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "OneSocketHT fail",
|
name: "OneSocketHT fail",
|
||||||
@ -185,22 +170,21 @@ func Test_Discover(t *testing.T) {
|
|||||||
Topology: []cadvisorapi.Node{
|
Topology: []cadvisorapi.Node{
|
||||||
{Id: 0,
|
{Id: 0,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{0, 4}},
|
{SocketID: 0, Id: 0, Threads: []int{0, 4}},
|
||||||
{Id: 1, Threads: []int{1, 5}},
|
{SocketID: 0, Id: 1, Threads: []int{1, 5}},
|
||||||
{Id: 2, Threads: []int{2, 6}},
|
{SocketID: 0, Id: 2, Threads: []int{2, 6}},
|
||||||
{Id: 3, Threads: []int{}}, // Wrong case - should fail here
|
{SocketID: 0, Id: 3, Threads: []int{}}, // Wrong case - should fail here
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
numaNodeInfo: NUMANodeInfo{},
|
want: &CPUTopology{},
|
||||||
want: &CPUTopology{},
|
wantErr: true,
|
||||||
wantErr: true,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
got, err := Discover(&tt.machineInfo, tt.numaNodeInfo)
|
got, err := Discover(&tt.machineInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if tt.wantErr {
|
if tt.wantErr {
|
||||||
t.Logf("Discover() expected error = %v", err)
|
t.Logf("Discover() expected error = %v", err)
|
||||||
|
@ -50,26 +50,21 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
Topology: []cadvisorapi.Node{
|
Topology: []cadvisorapi.Node{
|
||||||
{Id: 0,
|
{Id: 0,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{0, 6}},
|
{SocketID: 0, Id: 0, Threads: []int{0, 6}},
|
||||||
{Id: 1, Threads: []int{1, 7}},
|
{SocketID: 0, Id: 1, Threads: []int{1, 7}},
|
||||||
{Id: 2, Threads: []int{2, 8}},
|
{SocketID: 0, Id: 2, Threads: []int{2, 8}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{Id: 1,
|
{Id: 1,
|
||||||
Cores: []cadvisorapi.Core{
|
Cores: []cadvisorapi.Core{
|
||||||
{Id: 0, Threads: []int{3, 9}},
|
{SocketID: 1, Id: 0, Threads: []int{3, 9}},
|
||||||
{Id: 1, Threads: []int{4, 10}},
|
{SocketID: 1, Id: 1, Threads: []int{4, 10}},
|
||||||
{Id: 2, Threads: []int{5, 11}},
|
{SocketID: 1, Id: 2, Threads: []int{5, 11}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
numaNodeInfo := topology.NUMANodeInfo{
|
|
||||||
0: cpuset.NewCPUSet(0, 6, 1, 7, 2, 8),
|
|
||||||
1: cpuset.NewCPUSet(3, 9, 4, 10, 5, 11),
|
|
||||||
}
|
|
||||||
|
|
||||||
tcases := []struct {
|
tcases := []struct {
|
||||||
name string
|
name string
|
||||||
pod v1.Pod
|
pod v1.Pod
|
||||||
@ -237,7 +232,7 @@ func TestGetTopologyHints(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, tc := range tcases {
|
for _, tc := range tcases {
|
||||||
topology, _ := topology.Discover(&machineInfo, numaNodeInfo)
|
topology, _ := topology.Discover(&machineInfo)
|
||||||
|
|
||||||
var activePods []*v1.Pod
|
var activePods []*v1.Pod
|
||||||
for p := range tc.assignments {
|
for p := range tc.assignments {
|
||||||
|
@ -19,7 +19,6 @@ go_library(
|
|||||||
"//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
|
"//pkg/kubelet/apis/podresources/v1alpha1:go_default_library",
|
||||||
"//pkg/kubelet/checkpointmanager:go_default_library",
|
"//pkg/kubelet/checkpointmanager:go_default_library",
|
||||||
"//pkg/kubelet/checkpointmanager/errors:go_default_library",
|
"//pkg/kubelet/checkpointmanager/errors:go_default_library",
|
||||||
"//pkg/kubelet/cm/cpumanager/topology:go_default_library",
|
|
||||||
"//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library",
|
"//pkg/kubelet/cm/devicemanager/checkpoint:go_default_library",
|
||||||
"//pkg/kubelet/cm/topologymanager:go_default_library",
|
"//pkg/kubelet/cm/topologymanager:go_default_library",
|
||||||
"//pkg/kubelet/cm/topologymanager/bitmask:go_default_library",
|
"//pkg/kubelet/cm/topologymanager/bitmask:go_default_library",
|
||||||
@ -37,6 +36,7 @@ go_library(
|
|||||||
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
|
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
|
||||||
"//staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1:go_default_library",
|
"//staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1:go_default_library",
|
||||||
"//staging/src/k8s.io/kubelet/pkg/apis/pluginregistration/v1:go_default_library",
|
"//staging/src/k8s.io/kubelet/pkg/apis/pluginregistration/v1:go_default_library",
|
||||||
|
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||||
"//vendor/google.golang.org/grpc:go_default_library",
|
"//vendor/google.golang.org/grpc:go_default_library",
|
||||||
"//vendor/k8s.io/klog/v2:go_default_library",
|
"//vendor/k8s.io/klog/v2:go_default_library",
|
||||||
],
|
],
|
||||||
|
@ -26,6 +26,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
cadvisorapi "github.com/google/cadvisor/info/v1"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
|
|
||||||
@ -40,7 +41,6 @@ import (
|
|||||||
podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
|
podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
|
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors"
|
"k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors"
|
||||||
cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
|
"k8s.io/kubernetes/pkg/kubelet/cm/devicemanager/checkpoint"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/config"
|
"k8s.io/kubernetes/pkg/kubelet/config"
|
||||||
@ -124,11 +124,11 @@ func (s *sourcesReadyStub) AddSource(source string) {}
|
|||||||
func (s *sourcesReadyStub) AllReady() bool { return true }
|
func (s *sourcesReadyStub) AllReady() bool { return true }
|
||||||
|
|
||||||
// NewManagerImpl creates a new manager.
|
// NewManagerImpl creates a new manager.
|
||||||
func NewManagerImpl(numaNodeInfo cputopology.NUMANodeInfo, topologyAffinityStore topologymanager.Store) (*ManagerImpl, error) {
|
func NewManagerImpl(topology []cadvisorapi.Node, topologyAffinityStore topologymanager.Store) (*ManagerImpl, error) {
|
||||||
return newManagerImpl(pluginapi.KubeletSocket, numaNodeInfo, topologyAffinityStore)
|
return newManagerImpl(pluginapi.KubeletSocket, topology, topologyAffinityStore)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newManagerImpl(socketPath string, numaNodeInfo cputopology.NUMANodeInfo, topologyAffinityStore topologymanager.Store) (*ManagerImpl, error) {
|
func newManagerImpl(socketPath string, topology []cadvisorapi.Node, topologyAffinityStore topologymanager.Store) (*ManagerImpl, error) {
|
||||||
klog.V(2).Infof("Creating Device Plugin manager at %s", socketPath)
|
klog.V(2).Infof("Creating Device Plugin manager at %s", socketPath)
|
||||||
|
|
||||||
if socketPath == "" || !filepath.IsAbs(socketPath) {
|
if socketPath == "" || !filepath.IsAbs(socketPath) {
|
||||||
@ -136,8 +136,8 @@ func newManagerImpl(socketPath string, numaNodeInfo cputopology.NUMANodeInfo, to
|
|||||||
}
|
}
|
||||||
|
|
||||||
var numaNodes []int
|
var numaNodes []int
|
||||||
for node := range numaNodeInfo {
|
for _, node := range topology {
|
||||||
numaNodes = append(numaNodes, node)
|
numaNodes = append(numaNodes, node.Id)
|
||||||
}
|
}
|
||||||
|
|
||||||
dir, file := filepath.Split(socketPath)
|
dir, file := filepath.Split(socketPath)
|
||||||
|
@ -14,10 +14,10 @@ go_library(
|
|||||||
importpath = "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager",
|
importpath = "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager",
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
deps = [
|
deps = [
|
||||||
"//pkg/kubelet/cm/cpumanager/topology:go_default_library",
|
|
||||||
"//pkg/kubelet/cm/topologymanager/bitmask:go_default_library",
|
"//pkg/kubelet/cm/topologymanager/bitmask:go_default_library",
|
||||||
"//pkg/kubelet/lifecycle:go_default_library",
|
"//pkg/kubelet/lifecycle:go_default_library",
|
||||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||||
|
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||||
"//vendor/k8s.io/klog/v2:go_default_library",
|
"//vendor/k8s.io/klog/v2:go_default_library",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -20,9 +20,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
cadvisorapi "github.com/google/cadvisor/info/v1"
|
||||||
"k8s.io/api/core/v1"
|
"k8s.io/api/core/v1"
|
||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
|
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
|
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
|
||||||
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
|
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
|
||||||
)
|
)
|
||||||
@ -122,12 +122,12 @@ func (th *TopologyHint) LessThan(other TopologyHint) bool {
|
|||||||
var _ Manager = &manager{}
|
var _ Manager = &manager{}
|
||||||
|
|
||||||
//NewManager creates a new TopologyManager based on provided policy
|
//NewManager creates a new TopologyManager based on provided policy
|
||||||
func NewManager(numaNodeInfo cputopology.NUMANodeInfo, topologyPolicyName string) (Manager, error) {
|
func NewManager(topology []cadvisorapi.Node, topologyPolicyName string) (Manager, error) {
|
||||||
klog.Infof("[topologymanager] Creating topology manager with %s policy", topologyPolicyName)
|
klog.Infof("[topologymanager] Creating topology manager with %s policy", topologyPolicyName)
|
||||||
|
|
||||||
var numaNodes []int
|
var numaNodes []int
|
||||||
for node := range numaNodeInfo {
|
for _, node := range topology {
|
||||||
numaNodes = append(numaNodes, node)
|
numaNodes = append(numaNodes, node.Id)
|
||||||
}
|
}
|
||||||
|
|
||||||
if topologyPolicyName != PolicyNone && len(numaNodes) > maxAllowableNUMANodes {
|
if topologyPolicyName != PolicyNone && len(numaNodes) > maxAllowableNUMANodes {
|
||||||
|
Loading…
Reference in New Issue
Block a user